Search in sources :

Example 1 with REL_SUCCESS

use of org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_SUCCESS in project nifi by apache.

the class ConsumerLease method writeData.

private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) {
    FlowFile flowFile = session.create();
    final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding);
    tracker.incrementRecordCount(1);
    final byte[] value = record.value();
    if (value != null) {
        flowFile = session.write(flowFile, out -> {
            out.write(value);
        });
    }
    flowFile = session.putAllAttributes(flowFile, getAttributes(record));
    tracker.updateFlowFile(flowFile);
    populateAttributes(tracker);
    session.transfer(tracker.flowFile, REL_SUCCESS);
}
Also used : KafkaException(org.apache.kafka.common.KafkaException) HashMap(java.util.HashMap) ConsumerRecords(org.apache.kafka.clients.consumer.ConsumerRecords) ComponentLog(org.apache.nifi.logging.ComponentLog) ProcessException(org.apache.nifi.processor.exception.ProcessException) ArrayList(java.util.ArrayList) UTF8_ENCODING(org.apache.nifi.processors.kafka.pubsub.KafkaProcessorUtils.UTF8_ENCODING) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) ByteArrayInputStream(java.io.ByteArrayInputStream) Charset(java.nio.charset.Charset) RecordReader(org.apache.nifi.serialization.RecordReader) Map(java.util.Map) Record(org.apache.nifi.serialization.record.Record) OutputStream(java.io.OutputStream) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) HEX_ENCODING(org.apache.nifi.processors.kafka.pubsub.KafkaProcessorUtils.HEX_ENCODING) FlowFile(org.apache.nifi.flowfile.FlowFile) Collection(java.util.Collection) WriteResult(org.apache.nifi.serialization.WriteResult) REL_PARSE_FAILURE(org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_PARSE_FAILURE) IOException(java.io.IOException) ProcessSession(org.apache.nifi.processor.ProcessSession) RecordSetWriterFactory(org.apache.nifi.serialization.RecordSetWriterFactory) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) ConsumerRebalanceListener(org.apache.kafka.clients.consumer.ConsumerRebalanceListener) List(java.util.List) RecordReaderFactory(org.apache.nifi.serialization.RecordReaderFactory) REL_SUCCESS(org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_SUCCESS) Header(org.apache.kafka.common.header.Header) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Closeable(java.io.Closeable) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Pattern(java.util.regex.Pattern) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) DatatypeConverter(javax.xml.bind.DatatypeConverter) RecordSetWriter(org.apache.nifi.serialization.RecordSetWriter) InputStream(java.io.InputStream) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) FlowFile(org.apache.nifi.flowfile.FlowFile)

Aggregations

ByteArrayInputStream (java.io.ByteArrayInputStream)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 Charset (java.nio.charset.Charset)1 StandardCharsets (java.nio.charset.StandardCharsets)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1 TimeUnit (java.util.concurrent.TimeUnit)1 Pattern (java.util.regex.Pattern)1 Collectors (java.util.stream.Collectors)1 DatatypeConverter (javax.xml.bind.DatatypeConverter)1 Consumer (org.apache.kafka.clients.consumer.Consumer)1 ConsumerRebalanceListener (org.apache.kafka.clients.consumer.ConsumerRebalanceListener)1 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)1