Examples with Producer org.apache.kafka.clients.producer.Producer used on opensource projects

Example 1 with Producer

use of org.apache.kafka.clients.producer.Producer in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadTest method shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEosDisabled.

@Test
public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEosDisabled() {
    internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
    final StreamThread thread = createStreamThread(clientId, config, false);
    thread.setState(StreamThread.State.RUNNING);
    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    final List<TopicPartition> assignedPartitions = new ArrayList<>();
    // assign single partition
    assignedPartitions.add(t1p1);
    assignedPartitions.add(t1p2);
    activeTasks.put(task1, Collections.singleton(t1p1));
    activeTasks.put(task2, Collections.singleton(t1p2));
    thread.taskManager().setAssignmentMetadata(activeTasks, Collections.<TaskId, Set<TopicPartition>>emptyMap());
    final MockConsumer<byte[], byte[]> mockConsumer = (MockConsumer<byte[], byte[]>) thread.consumer;
    mockConsumer.assign(assignedPartitions);
    Map<TopicPartition, Long> beginOffsets = new HashMap<>();
    beginOffsets.put(t1p1, 0L);
    beginOffsets.put(t1p2, 0L);
    mockConsumer.updateBeginningOffsets(beginOffsets);
    thread.rebalanceListener.onPartitionsAssigned(new HashSet<>(assignedPartitions));
    assertEquals(1, clientSupplier.producers.size());
    final Producer globalProducer = clientSupplier.producers.get(0);
    for (final Task task : thread.tasks().values()) {
        assertSame(globalProducer, ((RecordCollectorImpl) ((StreamTask) task).recordCollector()).producer());
    }
    assertSame(clientSupplier.consumer, thread.consumer);
    assertSame(clientSupplier.restoreConsumer, thread.restoreConsumer);
}

Example 2 with Producer

use of org.apache.kafka.clients.producer.Producer in project nakadi by zalando.

the class KafkaTopicRepository method publishItem.

private static CompletableFuture<Exception> publishItem(final Producer<String, String> producer, final String topicId, final BatchItem item, final HystrixKafkaCircuitBreaker circuitBreaker) throws EventPublishingException {
    try {
        final CompletableFuture<Exception> result = new CompletableFuture<>();
        final ProducerRecord<String, String> kafkaRecord = new ProducerRecord<>(topicId, KafkaCursor.toKafkaPartition(item.getPartition()), item.getPartition(), item.dumpEventToString());
        circuitBreaker.markStart();
        producer.send(kafkaRecord, ((metadata, exception) -> {
            if (null != exception) {
                LOG.warn("Failed to publish to kafka topic {}", topicId, exception);
                item.updateStatusAndDetail(EventPublishingStatus.FAILED, "internal error");
                if (hasKafkaConnectionException(exception)) {
                    circuitBreaker.markFailure();
                } else {
                    circuitBreaker.markSuccessfully();
                }
                result.complete(exception);
            } else {
                item.updateStatusAndDetail(EventPublishingStatus.SUBMITTED, "");
                circuitBreaker.markSuccessfully();
                result.complete(null);
            }
        }));
        return result;
    } catch (final InterruptException e) {
        Thread.currentThread().interrupt();
        circuitBreaker.markSuccessfully();
        item.updateStatusAndDetail(EventPublishingStatus.FAILED, "internal error");
        throw new EventPublishingException("Error publishing message to kafka", e);
    } catch (final RuntimeException e) {
        circuitBreaker.markSuccessfully();
        item.updateStatusAndDetail(EventPublishingStatus.FAILED, "internal error");
        throw new EventPublishingException("Error publishing message to kafka", e);
    }
}

Example 3 with Producer

use of org.apache.kafka.clients.producer.Producer in project atlas by apache.

the class KafkaNotificationMockTest method shouldThrowExceptionIfProducerFails.

@Test
@SuppressWarnings("unchecked")
public void shouldThrowExceptionIfProducerFails() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenThrow(new RuntimeException("Simulating exception"));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    try {
        kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[] { message }));
        fail("Should have thrown NotificationException");
    } catch (NotificationException e) {
        assertEquals(e.getFailedMessages().size(), 1);
        assertEquals(e.getFailedMessages().get(0), "This is a test message");
    }
}

Example 4 with Producer

use of org.apache.kafka.clients.producer.Producer in project atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    TopicPartition topicPartition = new TopicPartition(topicName, 0);
    when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[] { message }));
    verify(producer).send(expectedRecord);
}

Example 5 with Producer

use of org.apache.kafka.clients.producer.Producer in project samza by apache.

the class TestKafkaSystemProducerJava method testInstantiateProducer.

@Test
public void testInstantiateProducer() {
    KafkaSystemProducer ksp = new KafkaSystemProducer("SysName", new ExponentialSleepStrategy(2.0, 200, 10000), new AbstractFunction0<Producer<byte[], byte[]>>() {

        @Override
        public Producer<byte[], byte[]> apply() {
            return new KafkaProducer<>(new HashMap<String, Object>());
        }
    }, new KafkaSystemProducerMetrics("SysName", new MetricsRegistryMap()), new AbstractFunction0<Object>() {

        @Override
        public Object apply() {
            return System.currentTimeMillis();
        }
    }, false);
    long now = System.currentTimeMillis();
    assertTrue((Long) ksp.clock().apply() >= now);
}