How to use the disdat.logger.info function in disdat

To help you get started, we’ve selected a few disdat examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kyocum / disdat / disdat / fs.py View on Github external
if not localize:
            all_keys = [k for k in all_keys if 'frame.pb' in k]
        fetch_count = 0
        fetch_tuples = []
        for s3_key in all_keys:
            obj_basename = os.path.basename(s3_key)
            obj_suffix = s3_key.replace(remote_obj_dir,'')
            obj_suffix_dir = os.path.dirname(obj_suffix).strip('/')  # remote_obj_dir won't have a trailing slash
            local_uuid_dir = os.path.join(data_context.get_object_dir(), obj_suffix_dir)
            local_object_path = os.path.join(local_uuid_dir, obj_basename)
            if not os.path.exists(local_object_path):
                fetch_count += 1
                fetch_tuples.append((s3_bucket, s3_key, local_object_path))
        _logger.info("Fast pull fetching {} objects...".format(fetch_count))
        results = aws_s3.get_s3_key_many(fetch_tuples)
        _logger.info("Fast pull completed {} transfers -- process pool closed and joined.".format(len(results)))
github kyocum / disdat / disdat / utility / aws_s3.py View on Github external
import botocore

    s3 = b3.resource('s3')
    exists = True
    try:
        s3.meta.client.head_bucket(Bucket=bucket)
    except botocore.exceptions.ClientError as e:
        error_code = int(e.response['Error']['Code'])
        if error_code == 404:
            exists = False
        elif error_code == 403:
            # for buckets you can get a forbidden instead of resource not found
            # if you have the s3:ListBucket permission on the bucket, Amazon S3 will return a
            # HTTP status code 404 ("no such key") error. If you don't have the s3:ListBucket permission,
            # Amazon S3 will return a HTTP status code 403 ("access denied") error.
            _logger.info("aws_s3: bucket {} raised a 403 (access forbidden), do you have ListBucket permission?".format(bucket))
            exists = False
        else:
            raise
    return exists
github kyocum / disdat / disdat / fs.py View on Github external
os.remove(os.path.join(DisdatConfig.instance().get_meta_dir(), META_FS_FILE))

        if local_context in self._all_contexts:
            dc = self._all_contexts[local_context]
            remote_context_url = dc.get_remote_object_dir()
            dc.delete_context(force=force)
            del self._all_contexts[local_context]

        if os.path.exists(ctxt_dir):
            shutil.rmtree(ctxt_dir)
            _logger.info("Disdat deleted local data context {}.".format(local_context))
            if remote:
                aws_s3.delete_s3_dir(remote_context_url)
                _logger.info("Disdat deleted remote data context {}.".format(remote_context_url))
        else:
            _logger.info("Disdat local data context {} appears to already have been deleted.".format(local_context))
github kyocum / disdat / disdat / data_context.py View on Github external
_logger.debug("Skipping a db file on bundle add")

                    elif o.scheme == 'file':
                        if dst_scheme == 's3':
                            # local to s3
                            aws_s3.put_s3_file(o.path, os.path.dirname(dst_file))
                        elif dst_scheme != 'db':  # assume 'file'
                            # local to local
                            shutil.copy(o.path, os.path.dirname(dst_file))
                        else:
                            raise Exception("copy_in_files: copy local file to unsupported scheme {}".format(dst_scheme))

                    else:
                        raise Exception("DataContext copy-in-file found bad scheme: {} from {}".format(o.scheme, o))
                else:
                    _logger.info("DataContext copy-in-file: Not adding files in directory {}".format(src_path))
            except (IOError, os.error) as why:
                _logger.error("Disdat add error: {} {} {}".format(src_path, dst_dir, str(why)))

        if return_one_file:
            return file_set[0]
        else:
            return file_set
github kyocum / disdat / disdat / hyperframe.py View on Github external
try:
            if type(pb_tbls) is dict:
                assert (isinstance(pb_rows, dict) or isinstance(pb_rows, defaultdict))
                results = []
                for k, tbl in pb_tbls.items(): # dict of tables
                    for r in pb_rows[k]:           # dict of list of rows
                        ins = tbl.insert()
                        results.append(db_conn.execute(ins, r))
                return results
            else:
                assert (type(pb_tbls) is not list)
                assert (type(pb_tbls) is not tuple)
                ins = pb_tbls.insert()
                return db_conn.execute(ins, pb_rows)
        except IntegrityError as ie:
            _logger.info("Writing class pb {} to table encountered error {}".format(self._pb_type(), ie))
            return None
github kyocum / disdat / disdat / data_context.py View on Github external
# At the moment, this is all or none.  There are cases where you could localize and pull
        # only some of the files, in which case we could mix local and remote.  However, that may
        # indicate that something else is wrong.   We should probably indicate that in the future.
        if all(os.path.isfile(lf) for lf in local_file_set):
            if strip_file_scheme:
                append = ''
            else:
                append = 'file://'
            file_set = ["{}{}".format(append, lf) for lf in local_file_set]
        else:
            # Note that remote_dir already includes the URL scheme
            remote_dir = self.get_remote_object_dir()
            if remote_dir is not None:
                file_set = [ "{}".format(os.path.join(remote_dir, fr.hframe_uuid, f.replace(common.BUNDLE_URI_SCHEME,''))) for f in urls]
            else:
                _logger.info("actualize_link_urls: Files are not local, and no remote context bound.")
                raise Exception("actualize_link_urls: Files are not local, and no remote context bound.")

        return file_set
github kyocum / disdat / disdat / fs.py View on Github external
Switch to a different local context.

        Args:
            local_context_name (str): May be / or 
            save (bool): Whether to record context change on disk.
        Returns:

        """
        assert local_context_name is not None

        repo, local_context = DisdatFS._parse_fq_context_name(local_context_name)

        if self.curr_context is not None and self.curr_context_name == local_context_name:
            assert(local_context in self._all_contexts)
            assert(self.curr_context == self._all_contexts[local_context_name])
            _logger.info("Disdat already within a valid data context_name {}".format(local_context))

        new_context = self.get_context(local_context_name)

        if new_context is not None:
            self.curr_context = new_context
            print("Switched to context {}".format(self.curr_context_name))
        else:
            print("In context {}".format(self.curr_context_name))