Examples with Callback org.apache.kafka.clients.producer.Callback used on opensource projects

Example 46 with Callback

use of org.apache.kafka.clients.producer.Callback in project flink by apache.

the class KafkaProducerTest method testPropagateExceptions.

@Test
@SuppressWarnings("unchecked")
public void testPropagateExceptions() {
    try {
        // mock kafka producer
        KafkaProducer<?, ?> kafkaProducerMock = mock(KafkaProducer.class);
        // partition setup
        when(kafkaProducerMock.partitionsFor(anyString())).thenReturn(// returning a unmodifiable list to mimic KafkaProducer#partitionsFor() behaviour
        Collections.singletonList(new PartitionInfo("mock_topic", 42, null, null, null)));
        // failure when trying to send an element
        when(kafkaProducerMock.send(any(ProducerRecord.class), any(Callback.class))).thenAnswer(new Answer<Future<RecordMetadata>>() {

            @Override
            public Future<RecordMetadata> answer(InvocationOnMock invocation) throws Throwable {
                Callback callback = (Callback) invocation.getArguments()[1];
                callback.onCompletion(null, new Exception("Test error"));
                return null;
            }
        });
        // make sure the FlinkKafkaProducer instantiates our mock producer
        whenNew(KafkaProducer.class).withAnyArguments().thenReturn(kafkaProducerMock);
        // (1) producer that propagates errors
        FlinkKafkaProducer09<String> producerPropagating = new FlinkKafkaProducer09<>("mock_topic", new SimpleStringSchema(), FakeStandardProducerConfig.get(), null);
        OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink(producerPropagating));
        testHarness.open();
        try {
            testHarness.processElement(new StreamRecord<>("value"));
            testHarness.processElement(new StreamRecord<>("value"));
            fail("This should fail with an exception");
        } catch (Exception e) {
            assertNotNull(e.getCause());
            assertNotNull(e.getCause().getMessage());
            assertTrue(e.getCause().getMessage().contains("Test error"));
        }
        // (2) producer that only logs errors
        FlinkKafkaProducer09<String> producerLogging = new FlinkKafkaProducer09<>("mock_topic", new SimpleStringSchema(), FakeStandardProducerConfig.get(), null);
        producerLogging.setLogFailuresOnly(true);
        testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink(producerLogging));
        testHarness.open();
        testHarness.processElement(new StreamRecord<>("value"));
        testHarness.processElement(new StreamRecord<>("value"));
        testHarness.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 47 with Callback

use of org.apache.kafka.clients.producer.Callback in project nifi by apache.

the class PublisherLease method publish.

protected void publish(final FlowFile flowFile, final Map<String, String> additionalAttributes, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
    final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent);
    addHeaders(flowFile, additionalAttributes, record);
    producer.send(record, new Callback() {

        @Override
        public void onCompletion(final RecordMetadata metadata, final Exception exception) {
            if (exception == null) {
                tracker.incrementAcknowledgedCount(flowFile);
            } else {
                tracker.fail(flowFile, exception);
                poison();
            }
        }
    });
    messagesSent.incrementAndGet();
    tracker.incrementSentCount(flowFile);
}

Example 48 with Callback

use of org.apache.kafka.clients.producer.Callback in project nifi by apache.

the class TestPublisherLease method testPoisonOnFailure.

@Test
@SuppressWarnings("unchecked")
public void testPoisonOnFailure() throws IOException {
    final AtomicInteger poisonCount = new AtomicInteger(0);
    final PublisherLease lease = new PublisherLease(producer, 1024 * 1024, 1000L, logger, true, null, StandardCharsets.UTF_8) {

        @Override
        public void poison() {
            poisonCount.incrementAndGet();
            super.poison();
        }
    };
    final FlowFile flowFile = new MockFlowFile(1L);
    final String topic = "unit-test";
    final byte[] messageKey = null;
    final byte[] demarcatorBytes = null;
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Callback callback = invocation.getArgumentAt(1, Callback.class);
            callback.onCompletion(null, new RuntimeException("Unit Test Intentional Exception"));
            return null;
        }
    }).when(producer).send(any(ProducerRecord.class), any(Callback.class));
    lease.publish(flowFile, new ByteArrayInputStream(new byte[1]), messageKey, demarcatorBytes, topic);
    assertEquals(1, poisonCount.get());
    final PublishResult result = lease.complete();
    assertTrue(result.isFailure());
}

Example 49 with Callback

use of org.apache.kafka.clients.producer.Callback in project nifi by apache.

the class PublisherLease method publish.

private void publish(final FlowFile flowFile, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
    final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent);
    producer.send(record, new Callback() {

        @Override
        public void onCompletion(final RecordMetadata metadata, final Exception exception) {
            if (exception == null) {
                tracker.incrementAcknowledgedCount(flowFile);
            } else {
                tracker.fail(flowFile, exception);
                poison();
            }
        }
    });
    tracker.incrementSentCount(flowFile);
}

Example 50 with Callback

use of org.apache.kafka.clients.producer.Callback in project camel by apache.

the class KafkaProducerTest method processAsyncSendsMessage.

@Test
public void processAsyncSendsMessage() throws Exception {
    endpoint.getConfiguration().setTopic("sometopic");
    Mockito.when(exchange.getIn()).thenReturn(in);
    Mockito.when(exchange.getOut()).thenReturn(out);
    in.setHeader(KafkaConstants.PARTITION_KEY, 4);
    producer.process(exchange, callback);
    ArgumentCaptor<Callback> callBackCaptor = ArgumentCaptor.forClass(Callback.class);
    Mockito.verify(producer.getKafkaProducer()).send(Matchers.any(ProducerRecord.class), callBackCaptor.capture());
    Callback kafkaCallback = callBackCaptor.getValue();
    kafkaCallback.onCompletion(new RecordMetadata(null, 1, 1), null);
    assertRecordMetadataExists();
}