Examples with SpecificRecordBase org.apache.avro.specific.SpecificRecordBase used on opensource projects

Example 1 with SpecificRecordBase

use of org.apache.avro.specific.SpecificRecordBase in project schema-registry by pravega.

the class SerializerTest method testAvroSerializers.

@Test
public void testAvroSerializers() {
    SchemaRegistryClient client = mock(SchemaRegistryClient.class);
    SerializerConfig config = SerializerConfig.builder().registryClient(client).groupId("groupId").build();
    AvroSchema<Test1> schema1 = AvroSchema.of(Test1.class);
    AvroSchema<Test2> schema2 = AvroSchema.of(Test2.class);
    VersionInfo versionInfo1 = new VersionInfo("name", SerializationFormat.Avro.getFullTypeName(), 0, 0);
    VersionInfo versionInfo2 = new VersionInfo("name", SerializationFormat.Avro.getFullTypeName(), 1, 1);
    doAnswer(x -> GroupProperties.builder().serializationFormat(SerializationFormat.Any).build()).when(client).getGroupProperties(anyString());
    doAnswer(x -> versionInfo1).when(client).getVersionForSchema(anyString(), eq(schema1.getSchemaInfo()));
    doAnswer(x -> versionInfo2).when(client).getVersionForSchema(anyString(), eq(schema2.getSchemaInfo()));
    doAnswer(x -> new EncodingId(0)).when(client).getEncodingId(anyString(), eq(versionInfo1), any());
    doAnswer(x -> new EncodingId(1)).when(client).getEncodingId(anyString(), eq(versionInfo2), any());
    doAnswer(x -> new EncodingInfo(versionInfo1, schema1.getSchemaInfo(), Codecs.None.getCodec().getCodecType())).when(client).getEncodingInfo(anyString(), eq(new EncodingId(0)));
    doAnswer(x -> new EncodingInfo(versionInfo2, schema2.getSchemaInfo(), Codecs.None.getCodec().getCodecType())).when(client).getEncodingInfo(anyString(), eq(new EncodingId(1)));
    doAnswer(x -> true).when(client).canReadUsing(anyString(), any());
    AvroSchema<Object> of = AvroSchema.of(SchemaDefinitions.ENUM);
    VersionInfo versionInfo3 = new VersionInfo(of.getSchema().getFullName(), SerializationFormat.Avro.getFullTypeName(), 0, 2);
    doAnswer(x -> versionInfo3).when(client).getVersionForSchema(anyString(), eq(of.getSchemaInfo()));
    doAnswer(x -> new EncodingId(2)).when(client).getEncodingId(anyString(), eq(versionInfo3), any());
    doAnswer(x -> new EncodingInfo(versionInfo3, of.getSchemaInfo(), Codecs.None.getCodec().getCodecType())).when(client).getEncodingInfo(anyString(), eq(new EncodingId(2)));
    Serializer<Object> serializerStr = AvroSerializerFactory.serializer(config, of);
    GenericData.EnumSymbol enumSymbol = new GenericData.EnumSymbol(of.getSchema(), "a");
    ByteBuffer serialized1 = serializerStr.serialize(enumSymbol);
    Serializer<Object> deserializer1 = AvroSerializerFactory.deserializer(config, of);
    Object deserializedEnum = deserializer1.deserialize(serialized1);
    assertEquals(deserializedEnum, enumSymbol);
    Serializer<Test1> serializer = AvroSerializerFactory.serializer(config, schema1);
    Test1 test1 = new Test1("name", 1);
    ByteBuffer serialized = serializer.serialize(test1);
    Serializer<Test1> deserializer = AvroSerializerFactory.deserializer(config, schema1);
    Test1 deserialized = deserializer.deserialize(serialized);
    assertEquals(deserialized, test1);
    serialized = serializer.serialize(test1);
    Serializer<Object> genericDeserializer = AvroSerializerFactory.genericDeserializer(config, null);
    Object genericDeserialized = genericDeserializer.deserialize(serialized);
    assertTrue(genericDeserialized instanceof GenericRecord);
    assertEquals(((GenericRecord) genericDeserialized).get("name").toString(), "name");
    assertEquals(((GenericRecord) genericDeserialized).get("field1"), 1);
    // multi type
    Test2 test2 = new Test2("name", 1, "2");
    AvroSchema<SpecificRecordBase> schema1Base = AvroSchema.ofSpecificRecord(Test1.class);
    AvroSchema<SpecificRecordBase> schema2Base = AvroSchema.ofSpecificRecord(Test2.class);
    Map<Class<? extends SpecificRecordBase>, AvroSchema<SpecificRecordBase>> map = new HashMap<>();
    map.put(Test1.class, schema1Base);
    map.put(Test2.class, schema2Base);
    Serializer<SpecificRecordBase> multiSerializer = AvroSerializerFactory.multiTypeSerializer(config, map);
    serialized = multiSerializer.serialize(test1);
    Serializer<SpecificRecordBase> multiDeserializer = AvroSerializerFactory.multiTypeDeserializer(config, map);
    SpecificRecordBase deserialized2 = multiDeserializer.deserialize(serialized);
    assertEquals(deserialized2, test1);
    serialized = multiSerializer.serialize(test2);
    deserialized2 = multiDeserializer.deserialize(serialized);
    assertEquals(deserialized2, test2);
    Map<Class<? extends SpecificRecordBase>, AvroSchema<SpecificRecordBase>> map2 = new HashMap<>();
    map2.put(Test1.class, schema1Base);
    Serializer<Either<SpecificRecordBase, Object>> fallbackDeserializer = AvroSerializerFactory.typedOrGenericDeserializer(config, map2);
    serialized = multiSerializer.serialize(test1);
    Either<SpecificRecordBase, Object> fallback = fallbackDeserializer.deserialize(serialized);
    assertTrue(fallback.isLeft());
    assertEquals(fallback.getLeft(), test1);
    serialized = multiSerializer.serialize(test2);
    fallback = fallbackDeserializer.deserialize(serialized);
    assertTrue(fallback.isRight());
}

Example 2 with SpecificRecordBase

use of org.apache.avro.specific.SpecificRecordBase in project pipelines by gbif.

the class GbifJsonConverter method getMultimediaConverter.

/**
 * String converter for {@link MultimediaRecord}, convert an object to specific string view * *
 *
 * <pre>{@code
 * Result example:
 *
 * {
 *  "id": "777",
 *  "multimediaItems": [
 *    {
 *      "type": "StillImage",
 *      "format": "image/jpeg",
 *      "identifier": "http://arctos.database.museum/media/10436011?open"
 *     },
 *     {
 *      "type": "MovingImage",
 *      "format": "video/mp4",
 *      "identifier": "http://arctos.database.museum/media/10436025?open"
 *      }
 *  ],
 *  "mediaTypes": [
 *     "StillImage",
 *     "MovingImage"
 *  ]
 *
 * }</pre>
 */
private BiConsumer<JsonConverter, SpecificRecordBase> getMultimediaConverter() {
    return (jc, record) -> {
        MultimediaRecord mr = (MultimediaRecord) record;
        if (!skipId) {
            jc.addJsonTextFieldNoCheck(Indexing.ID, mr.getId());
        }
        // multimedia items
        if (mr.getMultimediaItems() != null && !mr.getMultimediaItems().isEmpty()) {
            List<ObjectNode> items = mr.getMultimediaItems().stream().map(item -> {
                ObjectNode node = JsonConverter.createObjectNode();
                BiConsumer<String, String> addField = (k, v) -> Optional.ofNullable(v).filter(f -> !f.isEmpty()).ifPresent(r -> node.put(k, r));
                addField.accept("type", item.getType());
                addField.accept("format", item.getFormat());
                addField.accept("identifier", item.getIdentifier());
                addField.accept("audience", item.getAudience());
                addField.accept("contributor", item.getContributor());
                addField.accept("created", item.getCreated());
                addField.accept("creator", item.getCreator());
                addField.accept("description", item.getDescription());
                addField.accept("license", item.getLicense());
                addField.accept("publisher", item.getPublisher());
                addField.accept("references", item.getReferences());
                addField.accept("rightsHolder", item.getRightsHolder());
                addField.accept("source", item.getSource());
                addField.accept("title", item.getTitle());
                return node;
            }).collect(Collectors.toList());
            jc.addJsonArray("multimediaItems", items);
            // media types
            Set<TextNode> mediaTypes = mr.getMultimediaItems().stream().filter(i -> !Strings.isNullOrEmpty(i.getType())).map(Multimedia::getType).map(TextNode::valueOf).collect(Collectors.toSet());
            jc.addJsonArray("mediaTypes", mediaTypes);
            // media licenses
            Set<TextNode> mediaLicenses = mr.getMultimediaItems().stream().filter(i -> !Strings.isNullOrEmpty(i.getLicense())).map(Multimedia::getLicense).map(TextNode::valueOf).collect(Collectors.toSet());
            jc.addJsonArray("mediaLicenses", mediaLicenses);
        }
    };
}

Example 3 with SpecificRecordBase

use of org.apache.avro.specific.SpecificRecordBase in project hudi by apache.

the class HoodieAvroUtils method rewriteRecord.

public static GenericRecord rewriteRecord(GenericRecord genericRecord, Schema newSchema, boolean copyOverMetaFields, GenericRecord fallbackRecord) {
    GenericRecord newRecord = new GenericData.Record(newSchema);
    boolean isSpecificRecord = genericRecord instanceof SpecificRecordBase;
    for (Schema.Field f : newSchema.getFields()) {
        if (!(isSpecificRecord && isMetadataField(f.name()))) {
            copyOldValueOrSetDefault(genericRecord, newRecord, f);
        }
        if (isMetadataField(f.name()) && copyOverMetaFields) {
            // if meta field exists in primary generic record, copy over.
            if (genericRecord.getSchema().getField(f.name()) != null) {
                copyOldValueOrSetDefault(genericRecord, newRecord, f);
            } else if (fallbackRecord != null && fallbackRecord.getSchema().getField(f.name()) != null) {
                // if not, try to copy from the fallback record.
                copyOldValueOrSetDefault(fallbackRecord, newRecord, f);
            }
        }
    }
    if (!GenericData.get().validate(newSchema, newRecord)) {
        throw new SchemaCompatibilityException("Unable to validate the rewritten record " + genericRecord + " against schema " + newSchema);
    }
    return newRecord;
}

Example 4 with SpecificRecordBase

use of org.apache.avro.specific.SpecificRecordBase in project kafka-tutorials by confluentinc.

the class MultiEventAvroProduceConsumeAppTest method testProduceAvroMultipleEvents.

@Test
@SuppressWarnings("unchecked")
public void testProduceAvroMultipleEvents() {
    KafkaAvroSerializer avroSerializer = new KafkaAvroSerializer();
    avroSerializer.configure(commonConfigs, false);
    MockProducer<String, SpecificRecordBase> mockAvroProducer = new MockProducer<String, SpecificRecordBase>(true, stringSerializer, (Serializer) avroSerializer);
    produceConsumeApp.produceAvroEvents(() -> mockAvroProducer, (String) commonConfigs.get("proto.topic"), produceConsumeApp.avroEvents());
    List<KeyValue<String, SpecificRecordBase>> expectedKeyValues = produceConsumeApp.avroEvents().stream().map((e -> KeyValue.pair((String) e.get("customer_id"), e))).collect(Collectors.toList());
    List<KeyValue<String, SpecificRecordBase>> actualKeyValues = mockAvroProducer.history().stream().map(this::toKeyValue).collect(Collectors.toList());
    assertThat(actualKeyValues, equalTo(expectedKeyValues));
}

Example 5 with SpecificRecordBase

use of org.apache.avro.specific.SpecificRecordBase in project kafka-tutorials by confluentinc.

the class MultiEventAvroProduceConsumeApp method consumeAvroEvents.

public void consumeAvroEvents(final Supplier<Consumer<String, SpecificRecordBase>> consumerSupplier, final String topic, final List<String> eventTracker) {
    try (Consumer<String, SpecificRecordBase> eventConsumer = consumerSupplier.get()) {
        eventConsumer.subscribe(Collections.singletonList(topic));
        while (keepConsumingAvro) {
            ConsumerRecords<String, SpecificRecordBase> consumerRecords = eventConsumer.poll(Duration.ofSeconds(1));
            consumerRecords.forEach(consumerRec -> {
                SpecificRecord avroRecord = consumerRec.value();
                if (avroRecord instanceof Purchase) {
                    Purchase purchase = (Purchase) avroRecord;
                    eventTracker.add(String.format("Avro Purchase event -> %s", purchase.getItem()));
                } else if (avroRecord instanceof PageView) {
                    PageView pageView = (PageView) avroRecord;
                    eventTracker.add(String.format("Avro Pageview event -> %s", pageView.getUrl()));
                } else {
                    LOG.error("Unexpected type - this shouldn't happen");
                }
            });
        }
    }
}