Examples with KeyedMessage kafka.producer.KeyedMessage used on opensource projects

Example 6 with KeyedMessage

use of kafka.producer.KeyedMessage in project pinot by linkedin.

the class StreamAvroIntoKafkaCommand method execute.

@Override
public boolean execute() throws IOException {
    int messageDelayMillis = Integer.parseInt(_millisBetweenMessages);
    final boolean sleepRequired = 0 < messageDelayMillis;
    if (sleepRequired) {
        LOGGER.info("Streaming Avro file into Kafka topic {} with {} ms between messages", _kafkaTopic, _millisBetweenMessages);
    } else {
        LOGGER.info("Streaming Avro file into Kafka topic {} with no delay between messages", _kafkaTopic);
    }
    // Create Kafka producer
    Properties properties = new Properties();
    properties.put("metadata.broker.list", _kafkaBrokerList);
    properties.put("serializer.class", "kafka.serializer.DefaultEncoder");
    properties.put("request.required.acks", "1");
    ProducerConfig producerConfig = new ProducerConfig(properties);
    Producer<byte[], byte[]> producer = new Producer<byte[], byte[]>(producerConfig);
    try {
        // Open the Avro file
        DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(new File(_avroFile));
        // Iterate over every record
        for (GenericRecord genericRecord : reader) {
            // Write the message to Kafka
            String recordJson = genericRecord.toString();
            byte[] bytes = recordJson.getBytes("utf-8");
            KeyedMessage<byte[], byte[]> data = new KeyedMessage<byte[], byte[]>(_kafkaTopic, Longs.toByteArray(HashUtil.hash64(bytes, bytes.length)), bytes);
            producer.send(data);
            // Sleep between messages
            if (sleepRequired) {
                Uninterruptibles.sleepUninterruptibly(messageDelayMillis, TimeUnit.MILLISECONDS);
            }
        }
        reader.close();
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
    savePID(System.getProperty("java.io.tmpdir") + File.separator + ".streamAvro.pid");
    return true;
}

Example 7 with KeyedMessage

use of kafka.producer.KeyedMessage in project presto by prestodb.

the class TestManySegments method startKafka.

@BeforeClass
public void startKafka() throws Exception {
    embeddedKafka = EmbeddedKafka.createEmbeddedKafka();
    embeddedKafka.start();
    topicName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_");
    Properties topicProperties = new Properties();
    topicProperties.setProperty("segment.bytes", "1048576");
    embeddedKafka.createTopics(1, 1, topicProperties, topicName);
    try (CloseableProducer<Long, Object> producer = embeddedKafka.createProducer()) {
        int jMax = 10_000;
        int iMax = 100_000 / jMax;
        for (long i = 0; i < iMax; i++) {
            ImmutableList.Builder<KeyedMessage<Long, Object>> builder = ImmutableList.builder();
            for (long j = 0; j < jMax; j++) {
                builder.add(new KeyedMessage<Long, Object>(topicName, i, ImmutableMap.of("id", Long.toString(i * iMax + j), "value", UUID.randomUUID().toString())));
            }
            producer.send(builder.build());
        }
    }
}

Example 8 with KeyedMessage

use of kafka.producer.KeyedMessage in project apex-malhar by apache.

the class POJOKafkaOutputOperator method processTuple.

/**
 * Write the incoming tuple to Kafka
 * @param tuple incoming tuple
 */
protected void processTuple(Object tuple) {
    // Get the getter method from the keyField
    if (keyMethod == null && keyField != "") {
        pojoClass = tuple.getClass();
        try {
            keyMethod = generateGetterForKeyField();
        } catch (NoSuchFieldException e) {
            throw new RuntimeException("Field " + keyField + " is invalid: " + e);
        }
    }
    // Convert the given tuple to KeyedMessage
    KeyedMessage msg;
    if (keyMethod != null) {
        msg = new KeyedMessage(getTopic(), keyMethod.get(tuple), tuple);
    } else {
        msg = new KeyedMessage(getTopic(), tuple, tuple);
    }
    getProducer().send(msg);
    messageCount++;
    if (tuple instanceof byte[]) {
        byteCount += ((byte[]) tuple).length;
    }
}