How to use the hdfs.__main__.parse_arg function in hdfs

To help you get started, we’ve selected a few hdfs examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mtth / hdfs / test / test_main.py View on Github external
def test_parse_float(self):
    eq_(parse_arg({'foo': '123.4'}, 'foo', float), 123.4)
github mtth / hdfs / test / test_main.py View on Github external
def test_parse_invalid(self):
    parse_arg({'foo': 'a'}, 'foo', int)
github mtth / hdfs / test / test_main.py View on Github external
def test_parse_int(self):
    eq_(parse_arg({'foo': '1'}, 'foo', int), 1)
    eq_(parse_arg({'foo': '1'}, 'foo', int, ','), 1)
github mtth / hdfs / test / test_main.py View on Github external
def test_parse_int_list(self):
    eq_(parse_arg({'foo': '1,'}, 'foo', int, ','), [1])
    eq_(parse_arg({'foo': '1,2'}, 'foo', int, ','), [1,2])
github mtth / hdfs / test / test_main.py View on Github external
def test_parse_int(self):
    eq_(parse_arg({'foo': '1'}, 'foo', int), 1)
    eq_(parse_arg({'foo': '1'}, 'foo', int, ','), 1)
github mtth / hdfs / hdfs / ext / avro / __main__.py View on Github external
overwrite=overwrite,
      schema=parse_arg(args, '--schema', loads),
      codec=args['--codec'],
    )
    with writer:
      records = (loads(line) for line in stdin)
      for record in records:
        writer.write(record)
  else:
    reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
    with reader:
      if args['schema']:
        stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
      elif args['read']:
        encoder = _Encoder()
        num = parse_arg(args, '--num', int)
        freq = parse_arg(args, '--freq', float)
        if freq:
          for record in reader:
            if random() <= freq:
              stdout.write(encoder.encode(record))
              stdout.write('\n')
        else:
          for record in islice(reader, num):
            stdout.write(encoder.encode(record))
            stdout.write('\n')
github mtth / hdfs / hdfs / ext / avro / __main__.py View on Github external
def main(argv=None, client=None, stdin=sys.stdin, stdout=sys.stdout):
  """Entry point.

  :param argv: Arguments list.
  :param client: For testing.

  """
  args = docopt(__doc__, argv=argv)
  if not client:
    client = configure_client('hdfscli-avro', args)
  elif args['--log']:
    raise HdfsError('Logging is only available when no client is specified.')
  overwrite = args['--force']
  parts = parse_arg(args, '--parts', int, ',')
  if args['write']:
    writer = AvroWriter(
      client,
      args['HDFS_PATH'],
      overwrite=overwrite,
      schema=parse_arg(args, '--schema', loads),
      codec=args['--codec'],
    )
    with writer:
      records = (loads(line) for line in stdin)
      for record in records:
        writer.write(record)
  else:
    reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
    with reader:
      if args['schema']:
github mtth / hdfs / hdfs / ext / avro / __main__.py View on Github external
schema=parse_arg(args, '--schema', loads),
      codec=args['--codec'],
    )
    with writer:
      records = (loads(line) for line in stdin)
      for record in records:
        writer.write(record)
  else:
    reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
    with reader:
      if args['schema']:
        stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
      elif args['read']:
        encoder = _Encoder()
        num = parse_arg(args, '--num', int)
        freq = parse_arg(args, '--freq', float)
        if freq:
          for record in reader:
            if random() <= freq:
              stdout.write(encoder.encode(record))
              stdout.write('\n')
        else:
          for record in islice(reader, num):
            stdout.write(encoder.encode(record))
            stdout.write('\n')
github mtth / hdfs / hdfs / ext / avro / __main__.py View on Github external
:param client: For testing.

  """
  args = docopt(__doc__, argv=argv)
  if not client:
    client = configure_client('hdfscli-avro', args)
  elif args['--log']:
    raise HdfsError('Logging is only available when no client is specified.')
  overwrite = args['--force']
  parts = parse_arg(args, '--parts', int, ',')
  if args['write']:
    writer = AvroWriter(
      client,
      args['HDFS_PATH'],
      overwrite=overwrite,
      schema=parse_arg(args, '--schema', loads),
      codec=args['--codec'],
    )
    with writer:
      records = (loads(line) for line in stdin)
      for record in records:
        writer.write(record)
  else:
    reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
    with reader:
      if args['schema']:
        stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
      elif args['read']:
        encoder = _Encoder()
        num = parse_arg(args, '--num', int)
        freq = parse_arg(args, '--freq', float)
        if freq: