How to use the luigi.configuration function in luigi

To help you get started, we’ve selected a few luigi examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github spotify / luigi / test / helpers.py View on Github external
        @functools.wraps(fun)
        def wrapper(*args, **kwargs):
            import luigi.configuration
            orig_conf = luigi.configuration.LuigiConfigParser.instance()
            new_conf = luigi.configuration.LuigiConfigParser()
            luigi.configuration.LuigiConfigParser._instance = new_conf
            orig_dict = {k: dict(orig_conf.items(k)) for k in orig_conf.sections()}
            new_dict = self._make_dict(orig_dict)
            for (section, settings) in six.iteritems(new_dict):
                new_conf.add_section(section)
                for (name, value) in six.iteritems(settings):
                    new_conf.set(section, name, value)
            try:
                return fun(*args, **kwargs)
            finally:
                luigi.configuration.LuigiConfigParser._instance = orig_conf
        return wrapper
github treasure-data / luigi-td / luigi_td / test_config.py View on Github external
def tearDown(self):
        os.environ.clear()
        os.environ.update(self.environ)
        if os.path.exists('client.cfg'):
            os.unlink('client.cfg')
            luigi.configuration.LuigiConfigParser._instance = None
github spotify / luigi / test / helpers.py View on Github external
        @functools.wraps(fun)
        def wrapper(*args, **kwargs):
            import luigi.configuration
            orig_conf = luigi.configuration.LuigiConfigParser.instance()
            new_conf = luigi.configuration.LuigiConfigParser()
            luigi.configuration.LuigiConfigParser._instance = new_conf
            orig_dict = {k: dict(orig_conf.items(k)) for k in orig_conf.sections()}
            new_dict = self._make_dict(orig_dict)
            for (section, settings) in six.iteritems(new_dict):
                new_conf.add_section(section)
                for (name, value) in six.iteritems(settings):
                    new_conf.set(section, name, value)
            try:
                return fun(*args, **kwargs)
            finally:
                luigi.configuration.LuigiConfigParser._instance = orig_conf
        return wrapper
github treasure-data / luigi-td / luigi_td / test_config.py View on Github external
def _config(self, values):
        with open('client.cfg', 'w') as f:
            f.write("[td]\n")
            for key, val in values.items():
                f.write("{0}: {1}\n".format(key, val))
        luigi.configuration.LuigiConfigParser._instance = None
github treasure-data / luigi-td / luigi_td / config.py View on Github external
def load_default(self):
        luigi_config = luigi.configuration.get_config()
        apikey = luigi_config.get('td', 'apikey', os.environ.get('TD_API_KEY'))
        endpoint = luigi_config.get('td', 'endpoint', DEFAULT_ENDPOINT)
        self.config = Config(apikey, endpoint=endpoint)
github spotify / luigi / luigi / contrib / mysqldb.py View on Github external
logger = logging.getLogger('luigi-interface')

try:
    import mysql.connector
    from mysql.connector import errorcode, Error
except ImportError:
    logger.warning("Loading MySQL module without the python package mysql-connector-python. \
       This will crash at runtime if MySQL functionality is used.")


class MySqlTarget(luigi.Target):
    """
    Target for a resource in MySql.
    """

    marker_table = luigi.configuration.get_config().get('mysql', 'marker-table', 'table_updates')

    def __init__(self, host, database, user, password, table, update_id, **cnx_kwargs):
        """
        Initializes a MySqlTarget instance.

        :param host: MySql server address. Possibly a host:port string.
        :type host: str
        :param database: database name.
        :type database: str
        :param user: database user
        :type user: str
        :param password: password for specified user.
        :type password: str
        :param update_id: an identifier for this data set.
        :type update_id: str
        :param cnx_kwargs: optional params for mysql connector constructor.
github RUNDSP / luigi-swf / luigi_swf / cw.py View on Github external
def cw_api_sleep():
    c = luigi.configuration.get_config()
    t = c.getfloat('swfscheduler', 'cw_api_sleep', 0.2)
    sleep(t)
github ooni / pipeline / pipeline / batch / domain_intelligence.py View on Github external
import logging
import datetime
import tempfile
import subprocess

from six.moves.urllib.parse import urlparse

from bs4 import BeautifulSoup
import requests

import luigi
from luigi.postgres import PostgresTarget

from .sql_tasks import RunQuery

config = luigi.configuration.get_config()

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('ooni-pipeline')


def download_citizen_lab_test_list():
    archive_url = "https://github.com/citizenlab/test-lists/archive/master.zip"
    output_directory = tempfile.mkdtemp()

    r = requests.get(archive_url, stream=True)
    with tempfile.NamedTemporaryFile(delete=False) as fw:
        zip_filename = fw.name
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                fw.write(chunk)
github spotify / luigi / luigi / contrib / hive.py View on Github external
def get_hive_syntax():
    return luigi.configuration.get_config().get('hive', 'release', 'cdh4')
github groupon / luigi-warehouse / luigi_warehouse / sources.py View on Github external
def __init__(self,cfg_name='postgres'):
        import psycopg2
        self.host = luigi.configuration.get_config().get(cfg_name, 'host')
        self.database = luigi.configuration.get_config().get(cfg_name, 'database')
        self.user = luigi.configuration.get_config().get(cfg_name, 'user')
        self.password = luigi.configuration.get_config().get(cfg_name, 'password')
        self.port = luigi.configuration.get_config().get(cfg_name, 'port')
        self.connection = self.__connect__()
        self.cursor = self.connection.cursor()