How to use the avro.schema function in avro

To help you get started, we’ve selected a few avro examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ga4gh / ga4gh-server / tests / unit / test_schemas.py View on Github external
def getInvalidValue(self, cls, fieldName):
        """
        Returns a value that should trigger a schema validation failure.
        """
        fieldType = self.getAvroSchema(cls, fieldName).type
        if isinstance(fieldType, avro.schema.UnionSchema):
            types = list(t.type for t in fieldType.schemas)
            val = self.instanceGenerator.generateInvalidateTypeValue(*types)
        else:
            val = self.instanceGenerator.generateInvalidateTypeValue(fieldType)
        return val
github ibm-cloud-architecture / refarch-kc / data_schemas / avro_test / utils / avroEDAUtils.py View on Github external
def getContainerEventSchema(schema_files_location):
  # Read all the schemas needed in order to produce the final Container Event Schema
  known_schemas = avro.schema.Names()
  container_status_schema = LoadAvsc(schema_files_location + "/container_status.avsc", known_schemas)
  container_event_payload_schema = LoadAvsc(schema_files_location + "/container_event_payload.avsc", known_schemas)
  container_event_type_schema = LoadAvsc(schema_files_location + "/container_event_type.avsc", known_schemas)
  container_event_schema = LoadAvsc(schema_files_location + "/container_event.avsc", known_schemas)
  return container_event_schema
github ga4gh / ga4gh-server / tests / unit / test_schemas.py View on Github external
if isinstance(typ, avro.schema.UnionSchema):
            t0 = typ.schemas[0]
            t1 = typ.schemas[1]
            if isinstance(t0, avro.schema.PrimitiveSchema):
                if t0.type == "null":
                    typ = t1
                elif t1.type == "null":
                    typ = t0
                else:
                    raise Exception(err)
        ret = None
        if isinstance(typ, avro.schema.MapSchema):
            ret = {"key": ["value1", "value2"]}
            if not isinstance(typ.values, avro.schema.ArraySchema):
                raise Exception(err)
        elif isinstance(typ, avro.schema.ArraySchema):
            if cls.isEmbeddedType(field.name):
                embeddedClass = cls.getEmbeddedType(field.name)
                ret = [self.getTypicalInstance(embeddedClass)]
            else:
                try:
                    ret = [self.typicalValueMap[typ.items.type]]
                except KeyError:
                    ret = [self.typicalValueMap[typ.items.type]]

        elif isinstance(typ, avro.schema.EnumSchema):
            ret = typ.symbols[0]
        elif isinstance(typ, avro.schema.RecordSchema):
            self.assertTrue(cls.isEmbeddedType(fieldName))
            embeddedClass = cls.getEmbeddedType(fieldName)
            ret = self.getTypicalInstance(embeddedClass)
        elif typ.type in self.typicalValueMap:
github darenr / python-kafka-elasticsearch / avroconsumer.py View on Github external
import io
import requests
import json
import sys

topic = 'simple'
 
# To consume messages
consumer = KafkaConsumer(topic,
                         group_id='test_group',
                         consumer_timeout_ms=1000,
                         auto_commit_enable=True,
                         auto_commit_interval_ms=30 * 1000,
                         auto_offset_reset='smallest',
                         bootstrap_servers=['slc08use.us.oracle.com:9092'])
schema = avro.schema.parse('''
{"type":"record","name":"test8","fields":[{"name":"id","type":"int"},{"name":"name","type":"string"}]}
''')
print schema

while True:
  try:
    for msg in consumer:
      print msg
      #bytes_reader = io.BytesIO(msg.value)
      #decoder = avro.io.BinaryDecoder(bytes_reader)
      #reader = avro.io.DatumReader(schema)
      #print("%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition,
       #                                  msg.offset, msg.key,
        #                                 reader.read(decoder)))
      #consumer.commit()
github NCI-GDC / gdcdatamodel / gdcdatamodel / bin / gen_toplevel_schema.py View on Github external
def load_file_list(file_list):
    known_schemata = avro.schema.Names()
    for filename in file_list:
        gen_schema = load_avsc(filename, known_schemata)

    return gen_schema
github apache / avro / lang / py3 / avro / protocol.py View on Github external
def __str__(self):
    return json.dumps(self.to_json(), cls=schema.MappingProxyEncoder)
github Yelp / schematizer / schematizer / models / avro_schema.py View on Github external
target_schema_type = schema.RecordSchema
    element_type = 'record'
    support_doc = True

    @property
    def key(self):
        return self.schema_obj.fullname

    @property
    def nested_schema_objects(self):
        return self.schema_obj.fields


class _FieldElement(_SchemaElement):

    target_schema_type = schema.Field
    element_type = 'field'
    support_doc = True

    @property
    def key(self):
        return AvroSchemaElement.compose_key(
            self.parent_key,
            self.schema_obj.name
        )

    @property
    def nested_schema_objects(self):
        return [self.schema_obj.type]


class _EnumSchemaElement(_SchemaElement):
github rbystrit / avro_gen / avrogen / protocol.py View on Github external
custom_imports = custom_imports or []

    if six.PY3:
        proto = protocol.Parse(protocol_json)
    else:
        proto = protocol.parse(protocol_json)

    schemas = []
    messages = []
    schema_names = set()
    request_names = set()

    known_types = set()
    for schema_idx, record_schema in enumerate(proto.types):
        if isinstance(record_schema, (schema.RecordSchema, schema.EnumSchema)):
            schemas.append((schema_idx, record_schema))
            known_types.add(clean_fullname(record_schema.fullname))

    for message in (six.itervalues(proto.messages) if six.PY2 else proto.messages):
        messages.append((message, message.request, message.response if isinstance(message.response, (
            schema.EnumSchema, schema.RecordSchema)) and clean_fullname(message.response.fullname) not in known_types else None))
        if isinstance(message.response, (schema.EnumSchema, schema.RecordSchema)):
            known_types.add(clean_fullname(message.response.fullname))

    namespaces = {}
    for schema_idx, record_schema in schemas:
        ns, name = ns_.split_fullname(clean_fullname(record_schema.fullname))
        if ns not in namespaces:
            namespaces[ns] = {'requests': [], 'records': [], 'responses': []}
        namespaces[ns]['records'].append((schema_idx, record_schema))
github rbystrit / avro_gen / avrogen / core_writer.py View on Github external
"""
    Gets a python type-hint for a given schema
    :param schema.Schema field_schema:
    :return: String containing python type hint
    """
    if use_logical_types and field_schema.props.get('logicalType'):
        from avrogen.logical import DEFAULT_LOGICAL_TYPES
        lt = DEFAULT_LOGICAL_TYPES.get(field_schema.props.get('logicalType'))
        if lt:
            return lt.typename()

    if isinstance(field_schema, schema.PrimitiveSchema):
        if field_schema.fullname == 'null':
            return ''
        return __PRIMITIVE_TYPE_MAPPING[field_schema.fullname].__name__
    elif isinstance(field_schema, schema.FixedSchema):
        return 'bytes'
    elif isinstance(field_schema, schema.NamedSchema):
        return 'SchemaClasses.' + field_schema.fullname + 'Class'
    elif isinstance(field_schema, schema.ArraySchema):
        return 'list[' + get_field_type_name(field_schema.items, use_logical_types) + ']'
    elif isinstance(field_schema, schema.MapSchema):
        return 'dict[str, ' + get_field_type_name(field_schema.values, use_logical_types) + ']'
    elif isinstance(field_schema, schema.UnionSchema):
        type_names = [get_field_type_name(x, use_logical_types) for x in field_schema.schemas if
                      get_field_type_name(x, use_logical_types)]
        if len(type_names) > 1:
            return ' | '.join(type_names)
        elif len(type_names) == 1:
            return type_names[0]
        return ''