Examples with ProducerRecord org.apache.kafka.clients.producer.ProducerRecord used on opensource projects

Example 61 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project drill by axbaretto.

the class KafkaMessageGenerator method populateAvroMsgIntoKafka.

public void populateAvroMsgIntoKafka(String topic, int numMsg) throws IOException {
    KafkaProducer<String, GenericRecord> producer = new KafkaProducer<String, GenericRecord>(producerProperties);
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(Resources.getResource("drill-avro-test.avsc").openStream());
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    Random rand = new Random();
    for (int i = 0; i < numMsg; ++i) {
        builder.set("key1", UUID.randomUUID().toString());
        builder.set("key2", rand.nextInt());
        builder.set("key3", rand.nextBoolean());
        List<Integer> list = Lists.newArrayList();
        list.add(rand.nextInt(100));
        list.add(rand.nextInt(100));
        list.add(rand.nextInt(100));
        builder.set("key5", list);
        Map<String, Double> map = Maps.newHashMap();
        map.put("key61", rand.nextDouble());
        map.put("key62", rand.nextDouble());
        builder.set("key6", map);
        Record producerRecord = builder.build();
        ProducerRecord<String, GenericRecord> record = new ProducerRecord<String, GenericRecord>(topic, producerRecord);
        producer.send(record);
    }
    producer.close();
}

Example 62 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project drill by axbaretto.

the class KafkaMessageGenerator method populateJsonMsgIntoKafka.

public void populateJsonMsgIntoKafka(String topic, int numMsg) throws InterruptedException, ExecutionException {
    KafkaProducer<String, String> producer = new KafkaProducer<String, String>(producerProperties);
    Random rand = new Random();
    try {
        for (int i = 0; i < numMsg; ++i) {
            JsonObject object = new JsonObject();
            object.addProperty("key1", UUID.randomUUID().toString());
            object.addProperty("key2", rand.nextInt());
            object.addProperty("key3", rand.nextBoolean());
            JsonArray element2 = new JsonArray();
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            element2.add(new JsonPrimitive(rand.nextInt(100)));
            object.add("key5", element2);
            JsonObject element3 = new JsonObject();
            element3.addProperty("key61", rand.nextDouble());
            element3.addProperty("key62", rand.nextDouble());
            object.add("key6", element3);
            ProducerRecord<String, String> message = new ProducerRecord<String, String>(topic, object.toString());
            logger.info("Publishing message : {}", message);
            Future<RecordMetadata> future = producer.send(message);
            logger.info("Committed offset of the message : {}", future.get().offset());
        }
    } catch (Throwable th) {
        logger.error(th.getMessage(), th);
        throw new DrillRuntimeException(th.getMessage(), th);
    } finally {
        if (producer != null) {
            producer.close();
        }
    }
}

Example 63 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project atlas by apache.

the class KafkaNotificationMockTest method shouldThrowExceptionIfProducerFails.

@Test
@SuppressWarnings("unchecked")
public void shouldThrowExceptionIfProducerFails() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenThrow(new RuntimeException("Simulating exception"));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    try {
        kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[] { message }));
        fail("Should have thrown NotificationException");
    } catch (NotificationException e) {
        assertEquals(e.getFailedMessages().size(), 1);
        assertEquals(e.getFailedMessages().get(0), "This is a test message");
    }
}

Example 64 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    TopicPartition topicPartition = new TopicPartition(topicName, 0);
    when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[] { message }));
    verify(producer).send(expectedRecord);
}

Example 65 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project drill by apache.

the class KafkaMessageGenerator method populateJsonMsgWithTimestamps.

public void populateJsonMsgWithTimestamps(String topic, int numMsg) throws ExecutionException, InterruptedException {
    try (KafkaProducer<String, String> producer = new KafkaProducer<>(producerProperties)) {
        int halfCount = numMsg / 2;
        for (PartitionInfo tpInfo : producer.partitionsFor(topic)) {
            for (int i = 1; i <= numMsg; ++i) {
                JsonObject object = new JsonObject();
                object.addProperty("stringKey", UUID.randomUUID().toString());
                object.addProperty("intKey", numMsg - i);
                object.addProperty("boolKey", i % 2 == 0);
                long timestamp = i < halfCount ? (halfCount - i) : i;
                ProducerRecord<String, String> message = new ProducerRecord<>(tpInfo.topic(), tpInfo.partition(), timestamp, "key" + i, object.toString());
                logger.info("Publishing message : {}", message);
                Future<RecordMetadata> future = producer.send(message);
                logger.info("Committed offset of the message : {}", future.get().offset());
            }
        }
    }
}