Examples with KafkaPublisherResult org.apache.nifi.processors.kafka.KafkaPublisher.KafkaPublisherResult used on opensource projects

Example 1 with KafkaPublisherResult

use of org.apache.nifi.processors.kafka.KafkaPublisher.KafkaPublisherResult in project nifi by apache.

the class PutKafka method doRendezvousWithKafka.

/**
 * Will rendezvous with {@link KafkaPublisher} after building
 * {@link PublishingContext} and will produce the resulting {@link FlowFile}.
 * The resulting FlowFile contains all required information to determine
 * if message publishing originated from the provided FlowFile has actually
 * succeeded fully, partially or failed completely (see
 * {@link #isFailedFlowFile(FlowFile)}.
 */
private FlowFile doRendezvousWithKafka(final FlowFile flowFile, final ProcessContext context, final ProcessSession session) {
    final AtomicReference<KafkaPublisherResult> publishResultRef = new AtomicReference<>();
    session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(InputStream contentStream) throws IOException {
            PublishingContext publishingContext = PutKafka.this.buildPublishingContext(flowFile, context, contentStream);
            KafkaPublisherResult result = null;
            try {
                result = PutKafka.this.kafkaResource.publish(publishingContext);
            } catch (final IllegalArgumentException e) {
                getLogger().error("Failed to publish {}, due to {}", new Object[] { flowFile, e }, e);
                result = new KafkaPublisherResult(0, -1);
            }
            publishResultRef.set(result);
        }
    });
    FlowFile resultFile = publishResultRef.get().isAllAcked() ? this.cleanUpFlowFileIfNecessary(flowFile, session) : session.putAllAttributes(flowFile, this.buildFailedFlowFileAttributes(publishResultRef.get().getLastMessageAcked(), flowFile, context));
    return resultFile;
}

Example 2 with KafkaPublisherResult

use of org.apache.nifi.processors.kafka.KafkaPublisher.KafkaPublisherResult in project nifi by apache.

the class KafkaPublisherTest method validateSuccessfulSendAsWhole.

@Test
public void validateSuccessfulSendAsWhole() throws Exception {
    InputStream contentStream = new ByteArrayInputStream("Hello Kafka".getBytes(StandardCharsets.UTF_8));
    String topicName = "validateSuccessfulSendAsWhole";
    Properties kafkaProperties = this.buildProducerProperties();
    KafkaPublisher publisher = new KafkaPublisher(kafkaProperties, mock(ComponentLog.class));
    PublishingContext publishingContext = new PublishingContext(contentStream, topicName);
    KafkaPublisherResult result = publisher.publish(publishingContext);
    assertEquals(0, result.getLastMessageAcked());
    assertEquals(1, result.getMessagesSent());
    contentStream.close();
    publisher.close();
    ConsumerIterator<byte[], byte[]> iter = this.buildConsumer(topicName);
    assertNotNull(iter.next());
    try {
        iter.next();
    } catch (ConsumerTimeoutException e) {
    // that's OK since this is the Kafka mechanism to unblock
    }
}

Example 3 with KafkaPublisherResult

use of org.apache.nifi.processors.kafka.KafkaPublisher.KafkaPublisherResult in project nifi by apache.

the class KafkaPublisherTest method validateSuccessfulSendAsDelimited.

@Test
public void validateSuccessfulSendAsDelimited() throws Exception {
    InputStream contentStream = new ByteArrayInputStream("Hello Kafka\nHello Kafka\nHello Kafka\nHello Kafka\n".getBytes(StandardCharsets.UTF_8));
    String topicName = "validateSuccessfulSendAsDelimited";
    Properties kafkaProperties = this.buildProducerProperties();
    KafkaPublisher publisher = new KafkaPublisher(kafkaProperties, mock(ComponentLog.class));
    PublishingContext publishingContext = new PublishingContext(contentStream, topicName);
    publishingContext.setDelimiterBytes("\n".getBytes(StandardCharsets.UTF_8));
    KafkaPublisherResult result = publisher.publish(publishingContext);
    assertEquals(3, result.getLastMessageAcked());
    assertEquals(4, result.getMessagesSent());
    contentStream.close();
    publisher.close();
    ConsumerIterator<byte[], byte[]> iter = this.buildConsumer(topicName);
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    try {
        iter.next();
        fail();
    } catch (ConsumerTimeoutException e) {
    // that's OK since this is the Kafka mechanism to unblock
    }
}