Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 36 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project incubator-rya by apache.

the class KafkaRyaSubGraphExporter method export.

/**
 * Exports the RyaSubGraph to a Kafka topic equivalent to the result returned by {@link RyaSubGraph#getId()}
 * @param subgraph - RyaSubGraph exported to Kafka
 * @param contructID - rowID of result that is exported. Used for logging purposes.
 */
@Override
public void export(final String constructID, final RyaSubGraph subGraph) throws ResultExportException {
    checkNotNull(constructID);
    checkNotNull(subGraph);
    try {
        // Send the result to the topic whose name matches the PCJ ID.
        final ProducerRecord<String, RyaSubGraph> rec = new ProducerRecord<>(subGraph.getId(), subGraph);
        final Future<RecordMetadata> future = producer.send(rec);
        // Don't let the export return until the result has been written to the topic. Otherwise we may lose results.
        future.get();
        log.debug("Producer successfully sent record with id: {} and statements: {}", constructID, subGraph.getStatements());
    } catch (final Throwable e) {
        throw new ResultExportException("A result could not be exported to Kafka.", e);
    }
}

Example 37 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_partitionAdded_then_consumedFromBeginning.

@Test
public void when_partitionAdded_then_consumedFromBeginning() throws Exception {
    properties.setProperty("metadata.max.age.ms", "100");
    StreamKafkaP processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    produce(topic1Name, 0, "0");
    assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
    setPartitionCount(topic1Name, INITIAL_PARTITION_COUNT + 2);
    Thread.sleep(1000);
    // this allows production to the added partition
    resetProducer();
    boolean somethingInPartition1 = false;
    for (int i = 1; i < 11; i++) {
        Future<RecordMetadata> future = produce(topic1Name, i, Integer.toString(i));
        RecordMetadata recordMetadata = future.get();
        System.out.println("Entry " + i + " produced to partition " + recordMetadata.partition());
        somethingInPartition1 |= recordMetadata.partition() == 1;
    }
    assertTrue("nothing was produced to partition-1", somethingInPartition1);
    Set receivedEvents = new HashSet();
    for (int i = 1; i < 11; i++) {
        try {
            receivedEvents.add(consumeEventually(processor, outbox));
        } catch (AssertionError e) {
            throw new AssertionError("Unable to receive 10 items, events so far: " + receivedEvents);
        }
    }
    assertEquals(range(1, 11).mapToObj(i -> entry(i, Integer.toString(i))).collect(toSet()), receivedEvents);
}

Example 38 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project drill by axbaretto.

the class KafkaMessageGenerator method populateJsonMsgIntoKafka.

public void populateJsonMsgIntoKafka(String topic, int numMsg) throws InterruptedException, ExecutionException {
    KafkaProducer<String, String> producer = new KafkaProducer<String, String>(producerProperties);
    Random rand = new Random();
    try {
        for (int i = 0; i < numMsg; ++i) {
            JsonObject object = new JsonObject();
            object.addProperty("key1", UUID.randomUUID().toString());
            object.addProperty("key2", rand.nextInt());
            object.addProperty("key3", rand.nextBoolean());
            JsonArray element2 = new JsonArray();
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            object.add("key5", element2);
            JsonObject element3 = new JsonObject();
            element3.addProperty("key61", rand.nextDouble());
            element3.addProperty("key62", rand.nextDouble());
            object.add("key6", element3);
            ProducerRecord<String, String> message = new ProducerRecord<String, String>(topic, object.toString());
            logger.info("Publishing message : {}", message);
            Future<RecordMetadata> future = producer.send(message);
            logger.info("Committed offset of the message : {}", future.get().offset());
        }
    } catch (Throwable th) {
        logger.error(th.getMessage(), th);
        throw new DrillRuntimeException(th.getMessage(), th);
    } finally {
        if (producer != null) {
            producer.close();
        }
    }
}

Example 39 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project apache-kafka-on-k8s by banzaicloud.

the class ProducerBatch method completeFutureAndFireCallbacks.

private void completeFutureAndFireCallbacks(long baseOffset, long logAppendTime, RuntimeException exception) {
    // Set the future before invoking the callbacks as we rely on its state for the `onCompletion` call
    produceFuture.set(baseOffset, logAppendTime, exception);
    // execute callbacks
    for (Thunk thunk : thunks) {
        try {
            if (exception == null) {
                RecordMetadata metadata = thunk.future.value();
                if (thunk.callback != null)
                    thunk.callback.onCompletion(metadata, null);
            } else {
                if (thunk.callback != null)
                    thunk.callback.onCompletion(null, exception);
            }
        } catch (Exception e) {
            log.error("Error executing user-provided callback on message for topic-partition '{}'", topicPartition, e);
        }
    }
    produceFuture.done();
}

Example 40 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    TopicPartition topicPartition = new TopicPartition(topicName, 0);
    when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[] { message }));
    verify(producer).send(expectedRecord);
}