use of org.apache.druid.data.input.avro.AvroStreamInputFormat in project druid by druid-io.
the class AvroStreamInputFormatTest method testSerdeNonDefault.
@Test
public void testSerdeNonDefault() throws IOException {
Repository repository = new Avro1124RESTRepositoryClientWrapper("http://github.io");
AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), true, true);
NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
Assert.assertEquals(inputFormat, inputFormat2);
}
use of org.apache.druid.data.input.avro.AvroStreamInputFormat in project druid by druid-io.
the class AvroStreamInputFormatTest method testSerde.
@Test
public void testSerde() throws IOException {
Repository repository = new Avro1124RESTRepositoryClientWrapper("http://github.io");
AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), false, false);
NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
Assert.assertEquals(inputFormat, inputFormat2);
}
use of org.apache.druid.data.input.avro.AvroStreamInputFormat in project druid by druid-io.
the class AvroStreamInputFormatTest method testParse.
@Test
public void testParse() throws SchemaValidationException, IOException {
Repository repository = new InMemoryRepository(null);
AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), false, false);
NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
repository = ((SchemaRepoBasedAvroBytesDecoder) ((AvroStreamInputFormat) inputFormat2).getAvroBytesDecoder()).getSchemaRepository();
// prepare data
GenericRecord someAvroDatum = buildSomeAvroDatum();
// encode schema id
Avro1124SubjectAndIdConverter converter = new Avro1124SubjectAndIdConverter(TOPIC);
TypedSchemaRepository<Integer, Schema, String> repositoryClient = new TypedSchemaRepository<>(repository, new IntegerConverter(), new AvroSchemaConverter(), new IdentityConverter());
Integer id = repositoryClient.registerSchema(TOPIC, SomeAvroDatum.getClassSchema());
ByteBuffer byteBuffer = ByteBuffer.allocate(4);
converter.putSubjectAndId(id, byteBuffer);
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(byteBuffer.array());
// encode data
DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(someAvroDatum.getSchema());
// write avro datum to bytes
writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
final ByteEntity entity = new ByteEntity(ByteBuffer.wrap(out.toByteArray()));
InputRow inputRow = inputFormat2.createReader(new InputRowSchema(timestampSpec, dimensionsSpec, null), entity, null).read().next();
assertInputRowCorrect(inputRow, DIMENSIONS, false);
}
use of org.apache.druid.data.input.avro.AvroStreamInputFormat in project druid by druid-io.
the class AvroStreamInputFormatTest method testParseSchemaless.
@Test
public void testParseSchemaless() throws SchemaValidationException, IOException {
Repository repository = new InMemoryRepository(null);
AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), false, false);
NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
repository = ((SchemaRepoBasedAvroBytesDecoder) ((AvroStreamInputFormat) inputFormat2).getAvroBytesDecoder()).getSchemaRepository();
// prepare data
GenericRecord someAvroDatum = buildSomeAvroDatum();
// encode schema id
Avro1124SubjectAndIdConverter converter = new Avro1124SubjectAndIdConverter(TOPIC);
TypedSchemaRepository<Integer, Schema, String> repositoryClient = new TypedSchemaRepository<>(repository, new IntegerConverter(), new AvroSchemaConverter(), new IdentityConverter());
Integer id = repositoryClient.registerSchema(TOPIC, SomeAvroDatum.getClassSchema());
ByteBuffer byteBuffer = ByteBuffer.allocate(4);
converter.putSubjectAndId(id, byteBuffer);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
out.write(byteBuffer.array());
// encode data
DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(someAvroDatum.getSchema());
// write avro datum to bytes
writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
final ByteEntity entity = new ByteEntity(ByteBuffer.wrap(out.toByteArray()));
InputRow inputRow = inputFormat2.createReader(new InputRowSchema(timestampSpec, DimensionsSpec.EMPTY, null), entity, null).read().next();
assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS, false);
}
}
use of org.apache.druid.data.input.avro.AvroStreamInputFormat in project druid by druid-io.
the class AvroStreamInputFormatTest method testSerdeForSchemaRegistry.
@Test
public void testSerdeForSchemaRegistry() throws IOException {
AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRegistryBasedAvroBytesDecoder("http://test:8081", 100, null, null, null, null), false, false);
NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
Assert.assertEquals(inputFormat, inputFormat2);
}
Aggregations