How to use luigi - 10 common examples

import datetime

import luigi

from luigi_swf import util


class MyDependency(luigi.Task):

    dt = luigi.DateParameter()


class MyTask(luigi.Task):

    dt = luigi.DateParameter()

    def requires(self):
        return MyDependency(dt=self.dt)


def test_fullname():
    # Setup
    t = MyTask(dt=datetime.date(2050, 1, 1))

    # Execute
    modname, clsname = util.fullname(t)

    # Test
    assert modname == 'test_util'

from exaslct_src.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask
from exaslct_src.lib.base.docker_base_task import DockerBaseTask
from exaslct_src.lib.base.json_pickle_parameter import JsonPickleParameter
from exaslct_src.lib.data.container_info import ContainerInfo
from exaslct_src.lib.data.database_info import DatabaseInfo
from exaslct_src.lib.test_runner.container_log_thread import ContainerLogThread
from exaslct_src.lib.test_runner.database_credentials import DatabaseCredentialsParameter
from exaslct_src.lib.test_runner.is_database_ready_thread import IsDatabaseReadyThread


class WaitForTestDockerDatabase(DockerBaseTask, DatabaseCredentialsParameter):
    environment_name = luigi.Parameter()
    test_container_info = JsonPickleParameter(ContainerInfo, significant=False)  # type: ContainerInfo
    database_info = JsonPickleParameter(DatabaseInfo, significant=False)  # type: DatabaseInfo
    db_startup_timeout_in_seconds = luigi.IntParameter(10 * 60, significant=False)
    attempt = luigi.IntParameter(1)

    def run_task(self):
        test_container = self._client.containers.get(self.test_container_info.container_name)
        db_container_name = self.database_info.container_info.container_name
        db_container = self._client.containers.get(db_container_name)
        is_database_ready = \
            self.wait_for_database_startup(test_container, db_container)
        after_startup_db_log_file = self.get_log_path().joinpath("after_startup_db_log.tar.gz")
        self.save_db_log_files_as_gzip_tar(after_startup_db_log_file, db_container)
        self.return_object(is_database_ready)

    def wait_for_database_startup(self,
                                  test_container: Container,
                                  db_container: Container):
        container_log_thread, is_database_ready_thread = \
            self.start_wait_threads(db_container, test_container)

def _test_mode(self, mode, threshold=.5):
        from cluster_tools.thresholded_components import ThresholdedComponentsWorkflow
        task = ThresholdedComponentsWorkflow(tmp_folder=self.tmp_folder,
                                             config_dir=self.config_folder,
                                             target=self.target, max_jobs=self.max_jobs,
                                             input_path=self.input_path,
                                             input_key=self.input_key,
                                             output_path=self.output_path,
                                             output_key=self.output_key,
                                             assignment_key=self.assignment_key,
                                             threshold=threshold, threshold_mode=mode)
        ret = luigi.build([task], local_scheduler=True)
        self.assertTrue(ret)
        self._check_result(mode, threshold=threshold)

i = luigi.Parameter()

    def run(self):  # Use work instead of run to DEBUG
        logger.info('Running test job...')
        with open(self.output().path, 'w') as f:
            f.write('This is a test job.')
            f.close()

    def output(self):
        return luigi.LocalTarget(path=os.path.join(os.getcwd(), 'testjob_' + str(self.i) + '.txt'))


if __name__ == '__main__':
    tasks = [MyJobTask(i=str(i), select=i+1) for i in range(3)]
    luigi.build(tasks, local_scheduler=True, workers=3)

input_path=self.input_path,
                                     input_key=self.input_key,
                                     output_path=self.output_path,
                                     output_key=self.output_key,
                                     threshold=.5,
                                     dependency=DummyTask())
        offset_path = './tmp/offsets.json'
        with z5py.File(self.input_path) as f:
            shape = f[self.input_key].shape
        task = MergeOffsetsLocal(tmp_folder=self.tmp_folder,
                                 config_dir=self.config_folder,
                                 max_jobs=8,
                                 shape=shape,
                                 save_path=offset_path,
                                 dependency=task1)
        ret = luigi.build([task], local_scheduler=True)
        self.assertTrue(ret)
        self.assertTrue(os.path.exists(offset_path))

        # checks
        # load offsets from file
        with open(offset_path) as f:
            offsets_dict = json.load(f)
            offsets = offsets_dict['offsets']
            max_offset = int(offsets_dict['n_labels']) - 1

        # load output segmentation
        with z5py.File(self.output_path) as f:
            seg = f[self.output_key][:]

        blocking = nt.blocking([0, 0, 0], list(shape), self.block_shape)
        for block_id in range(blocking.numberOfBlocks):

def test_start_handler(self):
        saved_tasks = []

        @EmptyTask.event_handler(Event.START)
        def save_task(task):
            print("Saving task...")
            saved_tasks.append(task)

        t = EmptyTask(True)
        build([t], local_scheduler=True)
        self.assertEqual(saved_tasks, [t])

def test_region_features(self):
        from cluster_tools.features import RegionFeaturesWorkflow
        ret = luigi.build([RegionFeaturesWorkflow(input_path=self.input_path,
                                                  input_key=self.input_key,
                                                  labels_path=self.input_path,
                                                  labels_key=self.seg_key,
                                                  output_path=self.output_path,
                                                  output_key=self.output_key,
                                                  config_dir=self.config_folder,
                                                  tmp_folder=self.tmp_folder,
                                                  target=self.target,
                                                  max_jobs=self.max_jobs)],
                          local_scheduler=True)
        self.assertTrue(ret)
        feature_names = self.check_subresults()
        self.check_result(feature_names)

def setUp(self):
        self.fs = luigi.contrib.hdfs.get_autoconfig_client()
        cfg_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testconfig")
        hadoop_bin = os.path.join(os.environ['HADOOP_HOME'], 'bin/hadoop')
        cmd = "{} --config {}".format(hadoop_bin, cfg_path)
        self.stashed_hdfs_client = luigi.configuration.get_config().get('hadoop', 'command', None)
        luigi.configuration.get_config().set('hadoop', 'command', cmd)

def setUp(self):
        self.fs = luigi.contrib.hdfs.get_autoconfig_client()
        cfg_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testconfig")
        hadoop_bin = os.path.join(os.environ['HADOOP_HOME'], 'bin/hadoop')
        cmd = "{} --config {}".format(hadoop_bin, cfg_path)
        self.stashed_hdfs_client = luigi.configuration.get_config().get('hadoop', 'command', None)
        luigi.configuration.get_config().set('hadoop', 'command', cmd)

        @functools.wraps(fun)
        def wrapper(*args, **kwargs):
            import luigi.configuration
            orig_conf = luigi.configuration.LuigiConfigParser.instance()
            new_conf = luigi.configuration.LuigiConfigParser()
            luigi.configuration.LuigiConfigParser._instance = new_conf
            orig_dict = {k: dict(orig_conf.items(k)) for k in orig_conf.sections()}
            new_dict = self._make_dict(orig_dict)
            for (section, settings) in six.iteritems(new_dict):
                new_conf.add_section(section)
                for (name, value) in six.iteritems(settings):
                    new_conf.set(section, name, value)
            try:
                return fun(*args, **kwargs)
            finally:
                luigi.configuration.LuigiConfigParser._instance = orig_conf
        return wrapper