Examples with SchemaRegistryException org.apache.gobblin.kafka.schemareg.SchemaRegistryException used on opensource projects

Example 1 with SchemaRegistryException

use of org.apache.gobblin.kafka.schemareg.SchemaRegistryException in project incubator-gobblin by apache.

the class Kafka09DataWriterTest method testAvroSerialization.

@Test
public void testAvroSerialization() throws IOException, InterruptedException, SchemaRegistryException {
    String topic = "testAvroSerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", LiAvroSerializer.class.getName());
    // set up mock schema registry
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + KafkaSchemaRegistryConfigurationKeys.KAFKA_SCHEMA_REGISTRY_CLASS, ConfigDrivenMd5SchemaRegistry.class.getCanonicalName());
    Kafka09DataWriter<GenericRecord> kafka09DataWriter = new Kafka09DataWriter<>(props);
    WriteCallback callback = mock(WriteCallback.class);
    GenericRecord record = TestUtils.generateRandomAvroRecord();
    try {
        kafka09DataWriter.write(record, callback);
    } finally {
        kafka09DataWriter.close();
    }
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
    ConfigDrivenMd5SchemaRegistry schemaReg = new ConfigDrivenMd5SchemaRegistry(topic, record.getSchema());
    LiAvroDeserializer deser = new LiAvroDeserializer(schemaReg);
    GenericRecord receivedRecord = deser.deserialize(topic, message);
    Assert.assertEquals(record.toString(), receivedRecord.toString());
}

Example 2 with SchemaRegistryException

use of org.apache.gobblin.kafka.schemareg.SchemaRegistryException in project incubator-gobblin by apache.

the class LiAvroSerializerBase method serialize.

public byte[] serialize(String topic, GenericRecord data) throws SerializationException {
    Schema schema = data.getSchema();
    MD5Digest schemaId = null;
    try {
        schemaId = schemaRegistry.register(topic, schema);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        // MAGIC_BYTE | schemaId-bytes | avro_payload
        out.write(LiAvroSerDeHelper.MAGIC_BYTE);
        out.write(schemaId.asBytes());
        BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
        writer.write(data, encoder);
        encoder.flush();
        byte[] bytes = out.toByteArray();
        out.close();
        return bytes;
    } catch (IOException | SchemaRegistryException e) {
        throw new SerializationException(e);
    }
}

Example 3 with SchemaRegistryException

use of org.apache.gobblin.kafka.schemareg.SchemaRegistryException in project incubator-gobblin by apache.

the class LiAvroDeserializerBase method deserialize.

/**
 * @param topic topic associated with the data
 * @param data serialized bytes
 * @param outputSchema the schema to deserialize to. If null then the record schema is used.
 * @return deserialized object
 */
public GenericRecord deserialize(String topic, byte[] data, Schema outputSchema) throws SerializationException {
    try {
        if (data[0] != LiAvroSerDeHelper.MAGIC_BYTE) {
            throw new SerializationException(String.format("Unknown magic byte for topic: %s ", topic));
        }
        // read start after the first byte (magic byte)
        MD5Digest schemaId = MD5Digest.fromBytes(data, 1);
        Schema schema = _schemaRegistry.getById(schemaId);
        Decoder decoder = DecoderFactory.get().binaryDecoder(data, 1 + MD5Digest.MD5_BYTES_LENGTH, data.length - MD5Digest.MD5_BYTES_LENGTH - 1, null);
        _datumReader.setExpected(outputSchema);
        _datumReader.setSchema(schema);
        try {
            GenericRecord record = _datumReader.read(null, decoder);
            return record;
        } catch (IOException e) {
            log.error(String.format("Error during decoding record for topic %s: ", topic));
            throw e;
        }
    } catch (IOException | SchemaRegistryException e) {
        throw new SerializationException("Error during Deserialization", e);
    }
}

Example 4 with SchemaRegistryException

use of org.apache.gobblin.kafka.schemareg.SchemaRegistryException in project incubator-gobblin by apache.

the class Kafka08DataWriterTest method testAvroSerialization.

@Test
public void testAvroSerialization() throws IOException, InterruptedException, SchemaRegistryException {
    String topic = "testAvroSerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.gobblin.kafka.serialize.LiAvroSerializer");
    // set up mock schema registry
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + KafkaSchemaRegistryConfigurationKeys.KAFKA_SCHEMA_REGISTRY_CLASS, ConfigDrivenMd5SchemaRegistry.class.getCanonicalName());
    Kafka08DataWriter<GenericRecord> kafka08DataWriter = new Kafka08DataWriter<>(props);
    WriteCallback callback = mock(WriteCallback.class);
    GenericRecord record = TestUtils.generateRandomAvroRecord();
    try {
        kafka08DataWriter.write(record, callback);
    } finally {
        kafka08DataWriter.close();
    }
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
    ConfigDrivenMd5SchemaRegistry schemaReg = new ConfigDrivenMd5SchemaRegistry(topic, record.getSchema());
    LiAvroDeserializer deser = new LiAvroDeserializer(schemaReg);
    GenericRecord receivedRecord = deser.deserialize(topic, message);
    Assert.assertEquals(record.toString(), receivedRecord.toString());
}