Examples with ProducerRecord org.apache.kafka.clients.producer.ProducerRecord used on opensource projects

Example 96 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project hazelcast by hazelcast.

the class WriteKafkaPTest method testWriteToSpecificPartitions.

@Test
public void testWriteToSpecificPartitions() {
    String localTopic = topic;
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.map(sourceIMap)).writeTo(KafkaSinks.kafka(properties, e -> new ProducerRecord<>(localTopic, e.getKey(), e.getKey(), e.getValue())));
    instance().getJet().newJob(p).join();
    kafkaTestSupport.assertTopicContentsEventually(topic, sourceIMap, true);
}

Example 97 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project incubator-atlas by apache.

the class KafkaNotification method sendInternalToProducer.

@VisibleForTesting
void sendInternalToProducer(Producer p, NotificationType type, String[] messages) throws NotificationException {
    String topic = TOPIC_MAP.get(type);
    List<MessageContext> messageContexts = new ArrayList<>();
    for (String message : messages) {
        ProducerRecord record = new ProducerRecord(topic, message);
        LOG.debug("Sending message for topic {}: {}", topic, message);
        Future future = p.send(record);
        messageContexts.add(new MessageContext(future, message));
    }
    List<String> failedMessages = new ArrayList<>();
    Exception lastFailureException = null;
    for (MessageContext context : messageContexts) {
        try {
            RecordMetadata response = context.getFuture().get();
            LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset());
        } catch (Exception e) {
            lastFailureException = e;
            failedMessages.add(context.getMessage());
        }
    }
    if (lastFailureException != null) {
        throw new NotificationException(lastFailureException, failedMessages);
    }
}

Example 98 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project incubator-atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenReturn(new RecordMetadata(new TopicPartition(topicName, 0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, new String[] { message });
    verify(producer).send(expectedRecord);
}

Example 99 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project druid by druid-io.

the class KafkaSupervisorTest method addSomeEvents.

private void addSomeEvents(int numEventsPerPartition) throws Exception {
    // create topic manually
    try (Admin admin = kafkaServer.newAdminClient()) {
        admin.createTopics(Collections.singletonList(new NewTopic(topic, NUM_PARTITIONS, (short) 1))).all().get();
    }
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        kafkaProducer.initTransactions();
        kafkaProducer.beginTransaction();
        for (int i = 0; i < NUM_PARTITIONS; i++) {
            for (int j = 0; j < numEventsPerPartition; j++) {
                kafkaProducer.send(new ProducerRecord<>(topic, i, null, StringUtils.toUtf8(StringUtils.format("event-%d", j)))).get();
            }
        }
        kafkaProducer.commitTransaction();
    }
}

Example 100 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project druid by druid-io.

the class KafkaSupervisorTest method addMoreEvents.

private void addMoreEvents(int numEventsPerPartition, int num_partitions) throws Exception {
    try (Admin admin = kafkaServer.newAdminClient()) {
        admin.createPartitions(Collections.singletonMap(topic, NewPartitions.increaseTo(num_partitions))).all().get();
    }
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        kafkaProducer.initTransactions();
        kafkaProducer.beginTransaction();
        for (int i = NUM_PARTITIONS; i < num_partitions; i++) {
            for (int j = 0; j < numEventsPerPartition; j++) {
                kafkaProducer.send(new ProducerRecord<>(topic, i, null, StringUtils.toUtf8(StringUtils.format("event-%d", j)))).get();
            }
        }
        kafkaProducer.commitTransaction();
    }
}