Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 6 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project camel by apache.

the class KafkaProducerFullTest method producedStringMessageIsReceivedByKafka.

@Test
public void producedStringMessageIsReceivedByKafka() throws InterruptedException, IOException {
    int messageInTopic = 10;
    int messageInOtherTopic = 5;
    CountDownLatch messagesLatch = new CountDownLatch(messageInTopic + messageInOtherTopic);
    sendMessagesInRoute(messageInTopic, stringsTemplate, "IT test message", KafkaConstants.PARTITION_KEY, "1");
    sendMessagesInRoute(messageInOtherTopic, stringsTemplate, "IT test message in other topic", KafkaConstants.PARTITION_KEY, "1", KafkaConstants.TOPIC, TOPIC_STRINGS_IN_HEADER);
    createKafkaMessageConsumer(stringsConsumerConn, TOPIC_STRINGS, TOPIC_STRINGS_IN_HEADER, messagesLatch);
    boolean allMessagesReceived = messagesLatch.await(200, TimeUnit.MILLISECONDS);
    assertTrue("Not all messages were published to the kafka topics. Not received: " + messagesLatch.getCount(), allMessagesReceived);
    List<Exchange> exchangeList = mockEndpoint.getExchanges();
    assertEquals("Fifteen Exchanges are expected", exchangeList.size(), 15);
    for (Exchange exchange : exchangeList) {
        @SuppressWarnings("unchecked") List<RecordMetadata> recordMetaData1 = (List<RecordMetadata>) (exchange.getIn().getHeader(KafkaConstants.KAFKA_RECORDMETA));
        assertEquals("One RecordMetadata is expected.", recordMetaData1.size(), 1);
        assertTrue("Offset is positive", recordMetaData1.get(0).offset() >= 0);
        assertTrue("Topic Name start with 'test'", recordMetaData1.get(0).topic().startsWith("test"));
    }
}

Example 7 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project camel by apache.

the class KafkaProducerFullTest method producedString2MessageIsReceivedByKafka.

@Test
public void producedString2MessageIsReceivedByKafka() throws InterruptedException, IOException {
    int messageInTopic = 10;
    int messageInOtherTopic = 5;
    CountDownLatch messagesLatch = new CountDownLatch(messageInTopic + messageInOtherTopic);
    sendMessagesInRoute(messageInTopic, stringsTemplate2, "IT test message", (String[]) null);
    sendMessagesInRoute(messageInOtherTopic, stringsTemplate2, "IT test message in other topic", KafkaConstants.PARTITION_KEY, "1", KafkaConstants.TOPIC, TOPIC_STRINGS_IN_HEADER);
    createKafkaMessageConsumer(stringsConsumerConn, TOPIC_STRINGS, TOPIC_STRINGS_IN_HEADER, messagesLatch);
    boolean allMessagesReceived = messagesLatch.await(200, TimeUnit.MILLISECONDS);
    assertTrue("Not all messages were published to the kafka topics. Not received: " + messagesLatch.getCount(), allMessagesReceived);
    List<Exchange> exchangeList = mockEndpoint.getExchanges();
    assertEquals("Fifteen Exchanges are expected", exchangeList.size(), 15);
    for (Exchange exchange : exchangeList) {
        @SuppressWarnings("unchecked") List<RecordMetadata> recordMetaData1 = (List<RecordMetadata>) (exchange.getIn().getHeader(KafkaConstants.KAFKA_RECORDMETA));
        assertEquals("One RecordMetadata is expected.", recordMetaData1.size(), 1);
        assertTrue("Offset is positive", recordMetaData1.get(0).offset() >= 0);
        assertTrue("Topic Name start with 'test'", recordMetaData1.get(0).topic().startsWith("test"));
    }
}

Example 8 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project camel by apache.

the class KafkaProducerTest method processAsyncSendsMessageWithException.

@Test
public void processAsyncSendsMessageWithException() throws Exception {
    endpoint.getConfiguration().setTopic("sometopic");
    Mockito.when(exchange.getIn()).thenReturn(in);
    Mockito.when(exchange.getOut()).thenReturn(out);
    // setup the exception here
    org.apache.kafka.clients.producer.KafkaProducer kp = producer.getKafkaProducer();
    Mockito.when(kp.send(Matchers.any(ProducerRecord.class), Matchers.any(Callback.class))).thenThrow(new ApiException());
    in.setHeader(KafkaConstants.PARTITION_KEY, 4);
    producer.process(exchange, callback);
    ArgumentCaptor<Callback> callBackCaptor = ArgumentCaptor.forClass(Callback.class);
    Mockito.verify(producer.getKafkaProducer()).send(Matchers.any(ProducerRecord.class), callBackCaptor.capture());
    Mockito.verify(exchange).setException(Matchers.isA(ApiException.class));
    Mockito.verify(callback).done(Matchers.eq(true));
    Callback kafkaCallback = callBackCaptor.getValue();
    kafkaCallback.onCompletion(new RecordMetadata(null, 1, 1), null);
    assertRecordMetadataExists();
}

Example 9 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project open-kilda by telstra.

the class KafkaUtils method getStateDumpsFromBolts.

public DumpStateManager getStateDumpsFromBolts() {
    long timestamp = System.currentTimeMillis();
    String correlationId = String.format("atdd-%d", timestamp);
    CtrlRequest dumpRequest = new CtrlRequest("*", new RequestData("dump"), timestamp, correlationId, WFM_CTRL);
    try {
        RecordMetadata postedMessage = postMessage(settings.getControlTopic(), dumpRequest);
        KafkaConsumer<String, String> consumer = createConsumer();
        try {
            consumer.subscribe(Collections.singletonList(settings.getControlTopic()), new NoOpConsumerRebalanceListener() {

                @Override
                public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
                    System.out.println("Seek to offset: " + postedMessage.offset());
                    for (TopicPartition topicPartition : partitions) {
                        consumer.seek(topicPartition, postedMessage.offset());
                    }
                }
            });
            List<CtrlResponse> buffer = new ArrayList<>();
            final int BOLT_COUNT = 4;
            final int NUMBER_OF_ATTEMPTS = 5;
            int attempt = 0;
            while (buffer.size() < BOLT_COUNT && attempt++ < NUMBER_OF_ATTEMPTS) {
                for (ConsumerRecord<String, String> record : consumer.poll(1000)) {
                    System.out.println("Received message: (" + record.key() + ", " + record.value() + ") at offset " + record.offset());
                    Message message = MAPPER.readValue(record.value(), Message.class);
                    if (message.getDestination() == CTRL_CLIENT && message.getCorrelationId().equals(correlationId)) {
                        buffer.add((CtrlResponse) message);
                    }
                }
            }
            return DumpStateManager.fromResponsesList(buffer);
        } finally {
            consumer.close();
        }
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}

Example 10 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project streamsx.kafka by IBMStreams.

the class TransactionalKafkaProducerClient method commitTransaction.

@SuppressWarnings({ "rawtypes", "unchecked" })
private RecordMetadata commitTransaction(long sequenceId) throws Exception {
    ProducerRecord controlRecord = new ProducerRecord(EXACTLY_ONCE_STATE_TOPIC, null);
    Headers headers = controlRecord.headers();
    headers.add(TRANSACTION_ID, getTransactionalId().getBytes());
    headers.add(COMMITTED_SEQUENCE_ID, String.valueOf(sequenceId).getBytes());
    Future<RecordMetadata> controlRecordFuture = producer.send(controlRecord);
    if (logger.isDebugEnabled())
        logger.debug("Sent control record: " + controlRecord);
    if (logger.isDebugEnabled())
        logger.debug("Committing transaction...");
    producer.commitTransaction();
    // controlRecordFuture information should be available now
    RecordMetadata lastCommittedControlRecordMetadata = controlRecordFuture.get();
    return lastCommittedControlRecordMetadata;
}