Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
output_dir: string
directory where to write metadata.
duration: float
time taken by hashing input arguments, calling the wrapped
function and persisting its output.
args, kwargs: list and dict
input arguments for wrapped function
this_duration_limit: float
Max execution time for this function before issuing a warning.
"""
start_time = time.time()
argument_dict = filter_args(self.func, self.ignore,
args, kwargs)
input_repr = dict((k, repr(v)) for k, v in argument_dict.items())
# This can fail due to race-conditions with multiple
# concurrent joblibs removing the file or the directory
metadata = {"duration": duration, "input_args": input_repr}
func_id, args_id = self._get_output_identifiers(*args, **kwargs)
self.store_backend.store_metadata([func_id, args_id], metadata)
this_duration = time.time() - start_time
if this_duration > this_duration_limit:
# This persistence should be fast. It will not be if repr() takes
# time and its output is large, because json.dump will have to
# write a large file. This should not be an issue with numpy arrays
# for which repr() always output a short representation, but can
def get_output_dir(self, *args, **kwargs):
""" Returns the directory in which are persisted the results
of the function corresponding to the given arguments.
The results can be loaded using the .load_output method.
"""
coerce_mmap = (self.mmap_mode is not None)
argdict = filter_args(self.func, self.ignore,
args, kwargs)
for k,v in self.hashfun.iteritems():
if k in argdict:
argdict[k] = v(argdict[k])
#import ipdb; ipdb.set_trace()
argument_hash = hash(argdict,
coerce_mmap=coerce_mmap)
output_dir = os.path.join(self._get_func_dir(self.func),
argument_hash)
return output_dir, argument_hash
def get_output_key(self, args, kwargs):
"""Return the key that the output should be cached with, given
arguments, keyword arguments, and a list of arguments to ignore.
"""
# Get a dictionary mapping argument names to argument values where
# ignored arguments are omitted.
filtered_args = joblib.func_inspect.filter_args(
self.func, self.ignore, args, kwargs
)
# Get a sorted tuple of the filtered argument.
filtered_args = tuple(sorted(filtered_args.values()))
# Use native hash when hashing arguments.
return db.generate_key(filtered_args)
def _persist_input(self, output_dir, *args, **kwargs):
""" Save a small summary of the call using json format in the
output directory.
"""
argument_dict = filter_args(self.func, self.ignore,
args, kwargs)
input_repr = dict((k, repr(v)) for k, v in argument_dict.iteritems())
if json is not None:
# This can fail do to race-conditions with multiple
# concurrent joblibs removing the file or the directory
try:
mkdirp(output_dir)
json.dump(
input_repr,
file(os.path.join(output_dir, 'input_args.json'), 'w'),
)
except:
pass
return input_repr