How to use the joblib.disk.mkdirp function in joblib

To help you get started, we’ve selected a few joblib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github chrodan / tdlearn / joblib / View on Github external
def clear(self, warn=True):
        """ Empty the function's cache.
        func_dir = self._get_func_dir(mkdir=False)
        if self._verbose and warn:
            self.warn("Clearing cache %s" % func_dir)
        if os.path.exists(func_dir):
            shutil.rmtree(func_dir, ignore_errors=True)
        func_code, _, first_line = get_func_code(self.func)
        func_code_file = os.path.join(func_dir, '')
        self._write_func_code(func_code_file, func_code, first_line)
github chrodan / tdlearn / joblib / View on Github external
def _persist_output(self, output, dir):
        """ Persist the given output tuple in the directory.
            filename = os.path.join(dir, 'output.pkl')
            numpy_pickle.dump(output, filename, compress=self.compress)
            if self._verbose > 10:
                print 'Persisting in %s' % dir
        except OSError:
            " Race condition in the creation of the directory "
github chrodan / tdlearn / joblib / View on Github external
def _get_func_dir(self, mkdir=True):
        """ Get the directory corresponding to the cache for the
        module, name = get_func_name(self.func)
        func_dir = os.path.join(self.cachedir, *module)
        if mkdir:
        return func_dir
github chrodan / tdlearn / joblib / View on Github external
as functions are revaluated.
        # XXX: Bad explaination of the None value of cachedir
        self._verbose = verbose
        self.mmap_mode = mmap_mode
        self.timestamp = time.time()
        self.compress = compress
        if compress and mmap_mode is not None:
            warnings.warn('Compressed results cannot be memmapped',
        if cachedir is None:
            self.cachedir = None
            self.cachedir = os.path.join(cachedir, 'joblib')
github joblib / joblib / joblib / View on Github external
def __init__(self, logfile=None, logdir=None):
        if logfile is not None and logdir is not None:
            raise ValueError('Cannot specify both logfile and logdir')
        # XXX: Need argument docstring
        self.last_time = time.time()
        self.start_time = self.last_time
        if logdir is not None:
            logfile = os.path.join(logdir, 'joblib.log')
        self.logfile = logfile
        if logfile is not None:
            if os.path.exists(logfile):
                # Rotate the logs
                for i in range(1, 9):
                        shutil.move(logfile + '.%i' % i,
                                    logfile + '.%i' % (i + 1))
                        "No reason failing here"
                # Use a copy rather than a move, so that a process
                # monitoring this file does not get lost.
                    shutil.copy(logfile, logfile + '.1')
                    "No reason failing here"
                with open(logfile, 'w') as logfile:
github joblib / joblib / joblib / View on Github external
def configure(self, location, verbose=1, backend_options=None):
        """Configure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        if backend_options is None:
            backend_options = {}

        # setup location directory
        self.location = location
        if not os.path.exists(self.location):

        # item can be stored compressed for faster I/O
        self.compress = backend_options.get('compress', False)

        # FileSystemStoreBackend can be used with mmap_mode options under
        # certain conditions.
        mmap_mode = backend_options.get('mmap_mode')
        if self.compress and mmap_mode is not None:
            warnings.warn('Compressed items cannot be memmapped in a '
                          'filesystem store. Option will be ignored.',

        self.mmap_mode = mmap_mode
        self.verbose = verbose
github bmabey / provenance / examples / sftp / View on Github external
#!/usr/bin/env python

from joblib.disk import mkdirp

import provenance as p


                            '<            ""           remote machine                ""                       , ex. /home/me/artifacts>, you need to make sure that path directory exists.',