How to use joblibspark - 6 common examples

To help you get started, we’ve selected a few joblibspark examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github joblib / joblib-spark / test / test_backend.py View on Github external
def test_effective_n_jobs():

    backend = SparkDistributedBackend()
    max_num_concurrent_tasks = 8
    backend._get_max_num_concurrent_tasks = MagicMock(return_value=max_num_concurrent_tasks)

    assert backend.effective_n_jobs(n_jobs=None) == 1
    assert backend.effective_n_jobs(n_jobs=-1) == 8
    assert backend.effective_n_jobs(n_jobs=4) == 4

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        assert backend.effective_n_jobs(n_jobs=16) == 16
        assert len(w) == 1
github joblib / joblib-spark / joblibspark / backend.py View on Github external
def __init__(self, **backend_args):
        super(SparkDistributedBackend, self).__init__(**backend_args)
        self._pool = None
        self._n_jobs = None
        self._spark = SparkSession \
            .builder \
            .appName("JoblibSparkBackend") \
            .getOrCreate()
        self._job_group = "joblib-spark-job-group-" + str(uuid.uuid4())
github joblib / joblib-spark / joblibspark / backend.py View on Github external
def register():
    """
    Register joblib spark backend.
    """
    try:
        import sklearn  # pylint: disable=C0415
        if LooseVersion(sklearn.__version__) < LooseVersion('0.21'):
            warnings.warn("Your sklearn version is < 0.21, but joblib-spark only support "
                          "sklearn >=0.21 . You can upgrade sklearn to version >= 0.21 to "
                          "make sklearn use spark backend.")
    except ImportError:
        pass
    register_parallel_backend('spark', SparkDistributedBackend)
github joblib / joblib-spark / setup.py View on Github external
from setuptools import setup, find_packages

import joblibspark

if __name__ == '__main__':
    setup(name='joblibspark',
          version=joblibspark.__version__,
          author='Weichen Xu',
          author_email='weichen.xu@databricks.com',
          url='https://github.com/joblib/joblib-spark',
          description="Joblib Apache Spark Backend",
          long_description=joblibspark.__doc__,
          classifiers=[
              'Development Status :: 5 - Production/Stable',
              'Environment :: Console',
              'Intended Audience :: Developers',
              'Intended Audience :: Science/Research',
              'Intended Audience :: Education',
              'License :: OSI Approved :: Apache Software License',
              'Operating System :: OS Independent',
              'Programming Language :: Python :: 3',
              'Programming Language :: Python :: 3.6',
              'Programming Language :: Python :: 3.7',
              'Topic :: Scientific/Engineering',
              'Topic :: Utilities',
              'Topic :: Software Development :: Libraries',
          ],
          platforms='any',
github joblib / joblib-spark / setup.py View on Github external
from setuptools import setup, find_packages

import joblibspark

if __name__ == '__main__':
    setup(name='joblibspark',
          version=joblibspark.__version__,
          author='Weichen Xu',
          author_email='weichen.xu@databricks.com',
          url='https://github.com/joblib/joblib-spark',
          description="Joblib Apache Spark Backend",
          long_description=joblibspark.__doc__,
          classifiers=[
              'Development Status :: 5 - Production/Stable',
              'Environment :: Console',
              'Intended Audience :: Developers',
              'Intended Audience :: Science/Research',
              'Intended Audience :: Education',
              'License :: OSI Approved :: Apache Software License',
              'Operating System :: OS Independent',
              'Programming Language :: Python :: 3',
              'Programming Language :: Python :: 3.6',
              'Programming Language :: Python :: 3.7',
github joblib / joblib-spark / joblibspark / __init__.py View on Github external
def register_spark():
    """
    Register spark backend into joblib.
    """
    try:
        from .backend import register # pylint: disable=C0415
        register()
    except ImportError:
        msg = ("To use the spark.distributed backend you must install "
               "the pyspark and packages.\n\n")
        raise ImportError(msg)

joblibspark

Joblib Apache Spark Backend

Apache-2.0
Latest version published 11 months ago

Package Health Score

71 / 100
Full package analysis