Search in sources :

Example 1 with ReadOptions

use of org.apache.drill.exec.store.kafka.ReadOptions in project drill by apache.

the class AvroMessageReader method init.

@Override
public void init(SchemaNegotiator negotiator, ReadOptions readOptions, KafkaStoragePlugin plugin) {
    Properties kafkaConsumerProps = plugin.getConfig().getKafkaConsumerProps();
    Map<String, Object> propertiesMap = kafkaConsumerProps.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(), Map.Entry::getValue));
    deserializer = new KafkaAvroDeserializer(null, propertiesMap);
    TupleMetadata providedSchema = negotiator.providedSchema();
    loader = negotiator.build();
    AvroColumnConverterFactory factory = new AvroColumnConverterFactory(providedSchema);
    converter = factory.getRootConverter(providedSchema, new TupleSchema(), loader.writer());
    String keyDeserializer = kafkaConsumerProps.getProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG);
    deserializeKey = keyDeserializer != null && keyDeserializer.equals(KafkaAvroDeserializer.class.getName());
}
Also used : ColumnConverter(org.apache.drill.exec.record.ColumnConverter) SchemaNegotiator(org.apache.drill.exec.physical.impl.scan.framework.SchemaNegotiator) MetadataUtils(org.apache.drill.exec.record.metadata.MetadataUtils) UserException(org.apache.drill.common.exceptions.UserException) LoggerFactory(org.slf4j.LoggerFactory) RowSetLoader(org.apache.drill.exec.physical.resultSet.RowSetLoader) ReadOptions(org.apache.drill.exec.store.kafka.ReadOptions) Map(java.util.Map) ColumnMetadata(org.apache.drill.exec.record.metadata.ColumnMetadata) TupleSchema(org.apache.drill.exec.record.metadata.TupleSchema) ByteArrayDeserializer(org.apache.kafka.common.serialization.ByteArrayDeserializer) GenericRecord(org.apache.avro.generic.GenericRecord) AvroColumnConverterFactory(org.apache.drill.exec.store.avro.AvroColumnConverterFactory) Schema(org.apache.avro.Schema) Logger(org.slf4j.Logger) Properties(java.util.Properties) ResultSetLoader(org.apache.drill.exec.physical.resultSet.ResultSetLoader) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) IOException(java.io.IOException) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) Collectors(java.util.stream.Collectors) KafkaAvroDeserializer(io.confluent.kafka.serializers.KafkaAvroDeserializer) KafkaStoragePlugin(org.apache.drill.exec.store.kafka.KafkaStoragePlugin) TypeProtos(org.apache.drill.common.types.TypeProtos) MetaDataField(org.apache.drill.exec.store.kafka.MetaDataField) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) AvroColumnConverterFactory(org.apache.drill.exec.store.avro.AvroColumnConverterFactory) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) KafkaAvroDeserializer(io.confluent.kafka.serializers.KafkaAvroDeserializer) Properties(java.util.Properties) Map(java.util.Map) TupleSchema(org.apache.drill.exec.record.metadata.TupleSchema)

Aggregations

KafkaAvroDeserializer (io.confluent.kafka.serializers.KafkaAvroDeserializer)1 IOException (java.io.IOException)1 Map (java.util.Map)1 Properties (java.util.Properties)1 Collectors (java.util.stream.Collectors)1 Schema (org.apache.avro.Schema)1 GenericRecord (org.apache.avro.generic.GenericRecord)1 UserException (org.apache.drill.common.exceptions.UserException)1 TypeProtos (org.apache.drill.common.types.TypeProtos)1 SchemaNegotiator (org.apache.drill.exec.physical.impl.scan.framework.SchemaNegotiator)1 ResultSetLoader (org.apache.drill.exec.physical.resultSet.ResultSetLoader)1 RowSetLoader (org.apache.drill.exec.physical.resultSet.RowSetLoader)1 ColumnConverter (org.apache.drill.exec.record.ColumnConverter)1 ColumnMetadata (org.apache.drill.exec.record.metadata.ColumnMetadata)1 MetadataUtils (org.apache.drill.exec.record.metadata.MetadataUtils)1 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)1 TupleSchema (org.apache.drill.exec.record.metadata.TupleSchema)1 AvroColumnConverterFactory (org.apache.drill.exec.store.avro.AvroColumnConverterFactory)1 KafkaStoragePlugin (org.apache.drill.exec.store.kafka.KafkaStoragePlugin)1 MetaDataField (org.apache.drill.exec.store.kafka.MetaDataField)1