Examples with ProducerRecord org.apache.kafka.clients.producer.ProducerRecord used on opensource projects

Example 66 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project drill by apache.

the class KafkaMessageGenerator method populateAvroMsgIntoKafka.

public void populateAvroMsgIntoKafka(String topic, int numMsg) {
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
    try (KafkaProducer<Object, GenericRecord> producer = new KafkaProducer<>(producerProperties)) {
        Schema.Parser parser = new Schema.Parser();
        String userSchema = "{\"type\":\"record\"," + "\"name\":\"myrecord\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}," + "{\"name\":\"key2\",\"type\":\"int\"}," + "{\"name\":\"key3\",\"type\":\"boolean\"}," + "{\"name\":\"key5\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}," + "{\"name\":\"key6\",\"type\":{\"type\":\"record\",\"name\":\"myrecord6\",\"fields\":[" + "{\"name\":\"key61\",\"type\":\"double\"}," + "{\"name\":\"key62\",\"type\":\"double\"}]}}]}";
        Schema valueSchema = parser.parse(userSchema);
        GenericRecordBuilder valueBuilder = new GenericRecordBuilder(valueSchema);
        String key1Schema = "{\"type\":\"record\"," + "\"name\":\"key1record\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}]}\"";
        Schema keySchema = parser.parse(key1Schema);
        GenericRecordBuilder keyBuilder = new GenericRecordBuilder(keySchema);
        Random rand = new Random();
        for (int i = 0; i < numMsg; ++i) {
            // value record
            String key1 = UUID.randomUUID().toString();
            valueBuilder.set("key1", key1);
            valueBuilder.set("key2", rand.nextInt());
            valueBuilder.set("key3", rand.nextBoolean());
            List<Integer> list = Lists.newArrayList();
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            valueBuilder.set("key5", list);
            GenericRecordBuilder innerBuilder = new GenericRecordBuilder(valueSchema.getField("key6").schema());
            innerBuilder.set("key61", rand.nextDouble());
            innerBuilder.set("key62", rand.nextDouble());
            valueBuilder.set("key6", innerBuilder.build());
            Record producerRecord = valueBuilder.build();
            // key record
            keyBuilder.set("key1", key1);
            Record keyRecord = keyBuilder.build();
            ProducerRecord<Object, GenericRecord> record = new ProducerRecord<>(topic, keyRecord, producerRecord);
            producer.send(record);
        }
    }
}

Example 67 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project logging-log4j2 by apache.

the class KafkaAppenderTest method testAppendWithKeyLookup.

@Test
public void testAppendWithKeyLookup() throws Exception {
    final Appender appender = ctx.getRequiredAppender("KafkaAppenderWithKeyLookup");
    final LogEvent logEvent = createLogEvent();
    Date date = new Date();
    SimpleDateFormat format = new SimpleDateFormat("dd-MM-yyyy");
    appender.append(logEvent);
    final List<ProducerRecord<byte[], byte[]>> history = kafka.history();
    assertEquals(1, history.size());
    final ProducerRecord<byte[], byte[]> item = history.get(0);
    assertNotNull(item);
    assertEquals(TOPIC_NAME, item.topic());
    byte[] keyValue = format.format(date).getBytes(StandardCharsets.UTF_8);
    assertEquals(Long.valueOf(logEvent.getTimeMillis()), item.timestamp());
    assertArrayEquals(item.key(), keyValue);
    assertEquals(LOG_MESSAGE, new String(item.value(), StandardCharsets.UTF_8));
}

Example 68 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project logging-log4j2 by apache.

the class KafkaAppenderTest method testAppendWithLayout.

@Test
public void testAppendWithLayout() throws Exception {
    final Appender appender = ctx.getRequiredAppender("KafkaAppenderWithLayout");
    appender.append(createLogEvent());
    final List<ProducerRecord<byte[], byte[]>> history = kafka.history();
    assertEquals(1, history.size());
    final ProducerRecord<byte[], byte[]> item = history.get(0);
    assertNotNull(item);
    assertEquals(TOPIC_NAME, item.topic());
    assertNull(item.key());
    assertEquals("[" + LOG_MESSAGE + "]", new String(item.value(), StandardCharsets.UTF_8));
}

Example 69 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project flink by apache.

the class DynamicKafkaRecordSerializationSchema method serialize.

@Override
public ProducerRecord<byte[], byte[]> serialize(RowData consumedRow, KafkaSinkContext context, Long timestamp) {
    // shortcut in case no input projection is required
    if (keySerialization == null && !hasMetadata) {
        final byte[] valueSerialized = valueSerialization.serialize(consumedRow);
        return new ProducerRecord<>(topic, extractPartition(consumedRow, null, valueSerialized, context.getPartitionsForTopic(topic)), null, valueSerialized);
    }
    final byte[] keySerialized;
    if (keySerialization == null) {
        keySerialized = null;
    } else {
        final RowData keyRow = createProjectedRow(consumedRow, RowKind.INSERT, keyFieldGetters);
        keySerialized = keySerialization.serialize(keyRow);
    }
    final byte[] valueSerialized;
    final RowKind kind = consumedRow.getRowKind();
    if (upsertMode) {
        if (kind == RowKind.DELETE || kind == RowKind.UPDATE_BEFORE) {
            // transform the message as the tombstone message
            valueSerialized = null;
        } else {
            // make the message to be INSERT to be compliant with the INSERT-ONLY format
            final RowData valueRow = DynamicKafkaRecordSerializationSchema.createProjectedRow(consumedRow, kind, valueFieldGetters);
            valueRow.setRowKind(RowKind.INSERT);
            valueSerialized = valueSerialization.serialize(valueRow);
        }
    } else {
        final RowData valueRow = DynamicKafkaRecordSerializationSchema.createProjectedRow(consumedRow, kind, valueFieldGetters);
        valueSerialized = valueSerialization.serialize(valueRow);
    }
    return new ProducerRecord<>(topic, extractPartition(consumedRow, keySerialized, valueSerialized, context.getPartitionsForTopic(topic)), readMetadata(consumedRow, KafkaDynamicSink.WritableMetadata.TIMESTAMP), keySerialized, valueSerialized, readMetadata(consumedRow, KafkaDynamicSink.WritableMetadata.HEADERS));
}

Example 70 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project flink by apache.

the class KafkaTestBase method produceToKafka.

public static <K, V> void produceToKafka(Collection<ProducerRecord<K, V>> records, Class<? extends org.apache.kafka.common.serialization.Serializer<K>> keySerializerClass, Class<? extends org.apache.kafka.common.serialization.Serializer<V>> valueSerializerClass) throws Throwable {
    Properties props = new Properties();
    props.putAll(standardProps);
    props.putAll(kafkaServer.getIdempotentProducerConfig());
    props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializerClass.getName());
    props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializerClass.getName());
    AtomicReference<Throwable> sendingError = new AtomicReference<>();
    Callback callback = (metadata, exception) -> {
        if (exception != null) {
            if (!sendingError.compareAndSet(null, exception)) {
                sendingError.get().addSuppressed(exception);
            }
        }
    };
    try (KafkaProducer<K, V> producer = new KafkaProducer<>(props)) {
        for (ProducerRecord<K, V> record : records) {
            producer.send(record, callback);
        }
    }
    if (sendingError.get() != null) {
        throw sendingError.get();
    }
}