Search in sources :

Example 1 with REL_PARSE_FAILURE

use of org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_PARSE_FAILURE in project nifi by apache.

the class ConsumerLease method handleParseFailure.

private void handleParseFailure(final ConsumerRecord<byte[], byte[]> consumerRecord, final ProcessSession session, final Exception cause, final String message) {
    // If we are unable to parse the data, we need to transfer it to 'parse failure' relationship
    final Map<String, String> attributes = getAttributes(consumerRecord);
    attributes.put(KafkaProcessorUtils.KAFKA_OFFSET, String.valueOf(consumerRecord.offset()));
    attributes.put(KafkaProcessorUtils.KAFKA_PARTITION, String.valueOf(consumerRecord.partition()));
    attributes.put(KafkaProcessorUtils.KAFKA_TOPIC, consumerRecord.topic());
    FlowFile failureFlowFile = session.create();
    final byte[] value = consumerRecord.value();
    if (value != null) {
        failureFlowFile = session.write(failureFlowFile, out -> out.write(value));
    }
    failureFlowFile = session.putAllAttributes(failureFlowFile, attributes);
    final String transitUri = KafkaProcessorUtils.buildTransitURI(securityProtocol, bootstrapServers, consumerRecord.topic());
    session.getProvenanceReporter().receive(failureFlowFile, transitUri);
    session.transfer(failureFlowFile, REL_PARSE_FAILURE);
    if (cause == null) {
        logger.error(message);
    } else {
        logger.error(message, cause);
    }
    session.adjustCounter("Parse Failures", 1, false);
}
Also used : KafkaException(org.apache.kafka.common.KafkaException) HashMap(java.util.HashMap) ConsumerRecords(org.apache.kafka.clients.consumer.ConsumerRecords) ComponentLog(org.apache.nifi.logging.ComponentLog) ProcessException(org.apache.nifi.processor.exception.ProcessException) ArrayList(java.util.ArrayList) UTF8_ENCODING(org.apache.nifi.processors.kafka.pubsub.KafkaProcessorUtils.UTF8_ENCODING) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) ByteArrayInputStream(java.io.ByteArrayInputStream) Charset(java.nio.charset.Charset) RecordReader(org.apache.nifi.serialization.RecordReader) Map(java.util.Map) Record(org.apache.nifi.serialization.record.Record) OutputStream(java.io.OutputStream) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) HEX_ENCODING(org.apache.nifi.processors.kafka.pubsub.KafkaProcessorUtils.HEX_ENCODING) FlowFile(org.apache.nifi.flowfile.FlowFile) Collection(java.util.Collection) WriteResult(org.apache.nifi.serialization.WriteResult) REL_PARSE_FAILURE(org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_PARSE_FAILURE) IOException(java.io.IOException) ProcessSession(org.apache.nifi.processor.ProcessSession) RecordSetWriterFactory(org.apache.nifi.serialization.RecordSetWriterFactory) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) ConsumerRebalanceListener(org.apache.kafka.clients.consumer.ConsumerRebalanceListener) List(java.util.List) RecordReaderFactory(org.apache.nifi.serialization.RecordReaderFactory) REL_SUCCESS(org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0.REL_SUCCESS) Header(org.apache.kafka.common.header.Header) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Closeable(java.io.Closeable) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Pattern(java.util.regex.Pattern) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) DatatypeConverter(javax.xml.bind.DatatypeConverter) RecordSetWriter(org.apache.nifi.serialization.RecordSetWriter) InputStream(java.io.InputStream) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) FlowFile(org.apache.nifi.flowfile.FlowFile)

Aggregations

ByteArrayInputStream (java.io.ByteArrayInputStream)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 Charset (java.nio.charset.Charset)1 StandardCharsets (java.nio.charset.StandardCharsets)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1 TimeUnit (java.util.concurrent.TimeUnit)1 Pattern (java.util.regex.Pattern)1 Collectors (java.util.stream.Collectors)1 DatatypeConverter (javax.xml.bind.DatatypeConverter)1 Consumer (org.apache.kafka.clients.consumer.Consumer)1 ConsumerRebalanceListener (org.apache.kafka.clients.consumer.ConsumerRebalanceListener)1 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)1