How to use the datalab.storage._bucket.parse_name function in datalab

To help you get started, we’ve selected a few datalab examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github googledatalab / pydatalab / datalab / storage / commands / _storage.py View on Github external
def _storage_list(args, _):
  """ List the buckets or the contents of a bucket.

  This command is a bit different in that we allow wildchars in the bucket name and will list
  the buckets that match.
  """
  target = args['object'] if args['object'] else args['bucket']
  project = args['project']
  if target is None:
    return _storage_list_buckets(project, '*')  # List all buckets.

  bucket_name, key = datalab.storage._bucket.parse_name(target)
  if bucket_name is None:
    raise Exception('Cannot list %s; not a valid bucket name' % target)

  if key or not re.search('\?|\*|\[', target):
    # List the contents of the bucket
    if not key:
      key = '*'
    if project:
      # Only list if the bucket is in the project
      for bucket in datalab.storage.Buckets(project_id=project):
        if bucket.name == bucket_name:
          break
      else:
        raise Exception('%s does not exist in project %s' % (target, project))
    else:
      bucket = datalab.storage.Bucket(bucket_name)
github googledatalab / pydatalab / datalab / storage / _api.py View on Github external
def verify_permitted_to_read(gs_path):
    """Check if the user has permissions to read from the given path.

    Args:
      gs_path: the GCS path to check if user is permitted to read.
    Raises:
      Exception if user has no permissions to read.
    """
    # TODO(qimingj): Storage APIs need to be modified to allow absence of project
    #                or credential on Items. When that happens we can move the function
    #                to Items class.
    from . import _bucket
    bucket, prefix = _bucket.parse_name(gs_path)
    credentials = None
    if datalab.context.Context.is_signed_in():
      credentials = datalab.context._utils.get_credentials()
    args = {
        'maxResults': Api._MAX_RESULTS,
        'projection': 'noAcl'
    }
    if prefix is not None:
      args['prefix'] = prefix
    url = Api._ENDPOINT + (Api._OBJECT_PATH % (bucket, ''))
    try:
      datalab.utils.Http.request(url, args=args, credentials=credentials)
    except datalab.utils.RequestException as e:
      if e.status == 401:
        raise Exception('Not permitted to read from specified path. '
                        'Please sign in and make sure you have read access.')
github googledatalab / pydatalab / datalab / storage / commands / _storage.py View on Github external
def _get_item_contents(source_name):
  source_bucket, source_key = datalab.storage._bucket.parse_name(source_name)
  if source_bucket is None:
    raise Exception('Invalid source object name %s; no bucket specified.' % source_name)
  if source_key is None:
    raise Exception('Invalid source object name %si; source cannot be a bucket.' % source_name)
  source = datalab.storage.Item(source_bucket, source_key)
  if not source.exists():
    raise Exception('Source object %s does not exist' % source_name)
  return source.read_from()
github googledatalab / pydatalab / datalab / storage / commands / _storage.py View on Github external
def _storage_copy(args, _):
  target = args['destination']
  target_bucket, target_key = datalab.storage._bucket.parse_name(target)
  if target_bucket is None and target_key is None:
    raise Exception('Invalid copy target name %s' % target)

  sources = _expand_list(args['source'])

  if len(sources) > 1:
    # Multiple sources; target must be a bucket
    if target_bucket is None or target_key is not None:
      raise Exception('More than one source but target %s is not a bucket' % target)

  errs = []
  for source in sources:
    source_bucket, source_key = datalab.storage._bucket.parse_name(source)
    if source_bucket is None or source_key is None:
      raise Exception('Invalid source object name %s' % source)
    destination_bucket = target_bucket if target_bucket else source_bucket
github googledatalab / pydatalab / datalab / storage / commands / _storage.py View on Github external
def _expand_list(names):
  """ Do a wildchar name expansion of object names in a list and return expanded list.

    The items are expected to exist as this is used for copy sources or delete targets.
    Currently we support wildchars in the key name only.
  """

  if names is None:
    names = []
  elif isinstance(names, basestring):
    names = [names]

  results = []  # The expanded list.
  items = {}  # Cached contents of buckets; used for matching.
  for name in names:
    bucket, key = datalab.storage._bucket.parse_name(name)
    results_len = len(results)  # If we fail to add any we add name and let caller deal with it.
    if bucket:
      if not key:
        # Just a bucket; add it.
        results.append('gs://%s' % bucket)
      elif datalab.storage.Item(bucket, key).exists():
        results.append('gs://%s/%s' % (bucket, key))
      else:
        # Expand possible key values.
        if bucket not in items and key[:1] == '*':
          # We need the full list; cache a copy for efficiency.
          items[bucket] = [item.metadata.name
                           for item in list(datalab.storage.Bucket(bucket).items())]
        # If we have a cached copy use it
        if bucket in items:
          candidates = items[bucket]