How to use the gokart.object_storage.ObjectStorage function in gokart

To help you get started, we’ve selected a few gokart examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github m3dev / gokart / gokart / run.py View on Github external
def _try_to_delete_unnecessary_output_file(cmdline_args: List[str]):
    with CmdlineParser.global_instance(cmdline_args) as cp:
        task = cp.get_task_obj()  # type: gokart.TaskOnKart
        if task.delete_unnecessary_output_files:
            if ObjectStorage.if_object_storage_path(task.workspace_directory):
                logger.info('delete-unnecessary-output-files is not support s3/gcs.')
            else:
                gokart.delete_local_unnecessary_outputs(task)
            exit()
github m3dev / gokart / gokart / target.py View on Github external
def _make_file_system_target(file_path: str, processor: Optional[FileProcessor] = None) -> luigi.target.FileSystemTarget:
    processor = processor or make_file_processor(file_path)
    if ObjectStorage.if_object_storage_path(file_path):
        return ObjectStorage.get_object_storage_target(file_path, processor.format())
    return luigi.LocalTarget(file_path, format=processor.format())
github m3dev / gokart / gokart / target.py View on Github external
def _get_last_modification_time(path: str) -> datetime:
    if ObjectStorage.if_object_storage_path(path):
        if ObjectStorage.exists(path):
            return ObjectStorage.get_timestamp(path)
        raise FileNotFoundError(f'No such file or directory: {path}')
    return datetime.fromtimestamp(os.path.getmtime(path))
github m3dev / gokart / gokart / zip_client_util.py View on Github external
def make_zip_client(file_path: str, temporary_directory: str) -> ZipClient:
    if ObjectStorage.if_object_storage_path(file_path):
        return ObjectStorage.get_zip_client(file_path=file_path, temporary_directory=temporary_directory)
    return LocalZipClient(file_path=file_path, temporary_directory=temporary_directory)
github m3dev / gokart / gokart / zip_client_util.py View on Github external
def make_zip_client(file_path: str, temporary_directory: str) -> ZipClient:
    if ObjectStorage.if_object_storage_path(file_path):
        return ObjectStorage.get_zip_client(file_path=file_path, temporary_directory=temporary_directory)
    return LocalZipClient(file_path=file_path, temporary_directory=temporary_directory)
github m3dev / gokart / gokart / file_processor.py View on Github external
def load(self, file):
        if ObjectStorage.is_readable_objectstorage_instance(file):
            return pickle.loads(file.read())
        return pickle.load(_LargeLocalFileReader(file))
github m3dev / gokart / gokart / target.py View on Github external
def _get_last_modification_time(path: str) -> datetime:
    if ObjectStorage.if_object_storage_path(path):
        if ObjectStorage.exists(path):
            return ObjectStorage.get_timestamp(path)
        raise FileNotFoundError(f'No such file or directory: {path}')
    return datetime.fromtimestamp(os.path.getmtime(path))
github m3dev / gokart / gokart / target.py View on Github external
def _get_last_modification_time(path: str) -> datetime:
    if ObjectStorage.if_object_storage_path(path):
        if ObjectStorage.exists(path):
            return ObjectStorage.get_timestamp(path)
        raise FileNotFoundError(f'No such file or directory: {path}')
    return datetime.fromtimestamp(os.path.getmtime(path))