Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 81 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project druid by druid-io.

the class KafkaEventWriter method flush.

@Override
public void flush() {
    Exception e = null;
    for (Future<RecordMetadata> future : pendingWriteRecords) {
        try {
            future.get();
        } catch (InterruptedException | ExecutionException ex) {
            if (ex instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
            if (e == null) {
                e = ex;
            } else {
                e.addSuppressed(ex);
            }
        }
    }
    pendingWriteRecords.clear();
    if (e != null) {
        throw new RuntimeException(e);
    }
}

Example 82 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project hazelcast by hazelcast.

the class StreamKafkaPTest method produceEventToNewPartition.

private Entry<Integer, String> produceEventToNewPartition(int partitionId) throws Exception {
    String value;
    while (true) {
        // reset the producer for each attempt as it might not see the new partition yet
        kafkaTestSupport.resetProducer();
        value = UuidUtil.newUnsecureUuidString();
        Future<RecordMetadata> future = kafkaTestSupport.produce(topic1Name, partitionId, null, 0, value);
        RecordMetadata recordMetadata = future.get();
        if (recordMetadata.partition() == partitionId) {
            // if the event was added to the correct partition, stop
            break;
        }
        sleepMillis(250);
    }
    return entry(0, value);
}

Example 83 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project incubator-atlas by apache.

the class KafkaNotification method sendInternalToProducer.

@VisibleForTesting
void sendInternalToProducer(Producer p, NotificationType type, String[] messages) throws NotificationException {
    String topic = TOPIC_MAP.get(type);
    List<MessageContext> messageContexts = new ArrayList<>();
    for (String message : messages) {
        ProducerRecord record = new ProducerRecord(topic, message);
        LOG.debug("Sending message for topic {}: {}", topic, message);
        Future future = p.send(record);
        messageContexts.add(new MessageContext(future, message));
    }
    List<String> failedMessages = new ArrayList<>();
    Exception lastFailureException = null;
    for (MessageContext context : messageContexts) {
        try {
            RecordMetadata response = context.getFuture().get();
            LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset());
        } catch (Exception e) {
            lastFailureException = e;
            failedMessages.add(context.getMessage());
        }
    }
    if (lastFailureException != null) {
        throw new NotificationException(lastFailureException, failedMessages);
    }
}

Example 84 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project incubator-atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenReturn(new RecordMetadata(new TopicPartition(topicName, 0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, new String[] { message });
    verify(producer).send(expectedRecord);
}

Example 85 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project streamsx.kafka by IBMStreams.

the class TrackingProducerClient method waitForPermissionAndSendRecords.

private void waitForPermissionAndSendRecords(TupleProcessing pt) throws InterruptedException {
    final long tupleSeqNo = pt.getSeqNumber();
    if (trace.isTraceEnabled())
        trace.trace("processing tuple # " + tupleSeqNo);
    synchronized (pendingTuplesMonitor) {
        int n = 0;
        while (pendingTuples.size() >= maxPendingTuples) {
            if (n++ == 0)
                nQueueFullPause.increment();
            pendingTuplesMonitor.wait(10000L);
        }
    }
    synchronized (recoveryPendingMonitor) {
        // add pending tuple to the "queue". Must synchronize as the removal is done in a producer's callback thread
        if (trace.isTraceEnabled())
            trace.trace("queuing tuple # " + tupleSeqNo);
        synchronized (pendingTuples) {
            pendingTuples.put(tupleSeqNo, pt);
            nPendingTuples.setValue(pendingTuples.size());
        }
        if (trace.isTraceEnabled())
            trace.trace("queued tuple # " + tupleSeqNo);
        for (RecordProduceAttempt pr : pt.getPendingRecords()) {
            if (trace.isTraceEnabled())
                trace.trace("sending record # " + pr.getProducerRecordSeqNumber() + " @tuple # " + tupleSeqNo);
            try {
                Future<RecordMetadata> future = send(pr.getRecord(), pr.getCallback());
                pr.setFuture(future);
                if (trace.isTraceEnabled())
                    trace.trace("sent: record # " + pr.getProducerRecordSeqNumber() + " @tuple # " + tupleSeqNo);
            } catch (Exception e) {
                trace.warn("Failed sending record " + pr.getProducerRecordSeqNumber() + ": " + e.getMessage());
                initiateRecovery();
            }
        }
    }
}