Examples with ConsumerRecord org.apache.kafka.clients.consumer.ConsumerRecord used on opensource projects

Example 76 with ConsumerRecord

use of org.apache.kafka.clients.consumer.ConsumerRecord in project kafka by apache.

the class WorkerErrantRecordReporter method report.

@Override
public Future<Void> report(SinkRecord record, Throwable error) {
    ConsumerRecord<byte[], byte[]> consumerRecord;
    // report modified or new records, so handle both cases
    if (record instanceof InternalSinkRecord) {
        consumerRecord = ((InternalSinkRecord) record).originalRecord();
    } else {
        // Generate a new consumer record from the modified sink record. We prefer
        // to send the original consumer record (pre-transformed) to the DLQ,
        // but in this case we don't have one and send the potentially transformed
        // record instead
        String topic = record.topic();
        byte[] key = keyConverter.fromConnectData(topic, record.keySchema(), record.key());
        byte[] value = valueConverter.fromConnectData(topic, record.valueSchema(), record.value());
        RecordHeaders headers = new RecordHeaders();
        if (record.headers() != null) {
            for (Header header : record.headers()) {
                String headerKey = header.key();
                byte[] rawHeader = headerConverter.fromConnectHeader(topic, headerKey, header.schema(), header.value());
                headers.add(headerKey, rawHeader);
            }
        }
        int keyLength = key != null ? key.length : -1;
        int valLength = value != null ? value.length : -1;
        consumerRecord = new ConsumerRecord<>(record.topic(), record.kafkaPartition(), record.kafkaOffset(), record.timestamp(), record.timestampType(), keyLength, valLength, key, value, headers, Optional.empty());
    }
    Future<Void> future = retryWithToleranceOperator.executeFailed(Stage.TASK_PUT, SinkTask.class, consumerRecord, error);
    if (!future.isDone()) {
        TopicPartition partition = new TopicPartition(consumerRecord.topic(), consumerRecord.partition());
        futures.computeIfAbsent(partition, p -> new ArrayList<>()).add(future);
    }
    return future;
}

Example 77 with ConsumerRecord

use of org.apache.kafka.clients.consumer.ConsumerRecord in project kafka by apache.

the class MockConsumerInterceptor method onConsume.

@Override
public ConsumerRecords<String, String> onConsume(ConsumerRecords<String, String> records) {
    // This will ensure that we get the cluster metadata when onConsume is called for the first time
    // as subsequent compareAndSet operations will fail.
    CLUSTER_ID_BEFORE_ON_CONSUME.compareAndSet(NO_CLUSTER_ID, CLUSTER_META.get());
    Map<TopicPartition, List<ConsumerRecord<String, String>>> recordMap = new HashMap<>();
    for (TopicPartition tp : records.partitions()) {
        List<ConsumerRecord<String, String>> lst = new ArrayList<>();
        for (ConsumerRecord<String, String> record : records.records(tp)) {
            lst.add(new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), record.timestamp(), record.timestampType(), record.serializedKeySize(), record.serializedValueSize(), record.key(), record.value().toUpperCase(Locale.ROOT), new RecordHeaders(), Optional.empty()));
        }
        recordMap.put(tp, lst);
    }
    return new ConsumerRecords<>(recordMap);
}

Example 78 with ConsumerRecord

use of org.apache.kafka.clients.consumer.ConsumerRecord in project kafka by apache.

the class ErrorHandlingTaskTest method testErrorHandlingInSinkTasks.

@Test
public void testErrorHandlingInSinkTasks() throws Exception {
    Map<String, String> reportProps = new HashMap<>();
    reportProps.put(ConnectorConfig.ERRORS_LOG_ENABLE_CONFIG, "true");
    reportProps.put(ConnectorConfig.ERRORS_LOG_INCLUDE_MESSAGES_CONFIG, "true");
    LogReporter reporter = new LogReporter(taskId, connConfig(reportProps), errorHandlingMetrics);
    RetryWithToleranceOperator retryWithToleranceOperator = operator();
    retryWithToleranceOperator.metrics(errorHandlingMetrics);
    retryWithToleranceOperator.reporters(singletonList(reporter));
    createSinkTask(initialState, retryWithToleranceOperator);
    expectInitializeTask();
    expectTaskGetTopic(true);
    // valid json
    ConsumerRecord<byte[], byte[]> record1 = new ConsumerRecord<>(TOPIC, PARTITION1, FIRST_OFFSET, null, "{\"a\": 10}".getBytes());
    // bad json
    ConsumerRecord<byte[], byte[]> record2 = new ConsumerRecord<>(TOPIC, PARTITION2, FIRST_OFFSET, null, "{\"a\" 10}".getBytes());
    EasyMock.expect(consumer.poll(Duration.ofMillis(EasyMock.anyLong()))).andReturn(records(record1));
    EasyMock.expect(consumer.poll(Duration.ofMillis(EasyMock.anyLong()))).andReturn(records(record2));
    sinkTask.put(EasyMock.anyObject());
    EasyMock.expectLastCall().times(2);
    PowerMock.replayAll();
    workerSinkTask.initialize(TASK_CONFIG);
    workerSinkTask.initializeAndStart();
    workerSinkTask.iteration();
    workerSinkTask.iteration();
    // two records were consumed from Kafka
    assertSinkMetricValue("sink-record-read-total", 2.0);
    // only one was written to the task
    assertSinkMetricValue("sink-record-send-total", 1.0);
    // one record completely failed (converter issues)
    assertErrorHandlingMetricValue("total-record-errors", 1.0);
    // 2 failures in the transformation, and 1 in the converter
    assertErrorHandlingMetricValue("total-record-failures", 3.0);
    // one record completely failed (converter issues), and thus was skipped
    assertErrorHandlingMetricValue("total-records-skipped", 1.0);
    PowerMock.verifyAll();
}

Example 79 with ConsumerRecord

use of org.apache.kafka.clients.consumer.ConsumerRecord in project kafka by apache.

the class Fetcher method parseRecord.

/**
 * Parse the record entry, deserializing the key / value fields if necessary
 */
private ConsumerRecord<K, V> parseRecord(TopicPartition partition, RecordBatch batch, Record record) {
    try {
        long offset = record.offset();
        long timestamp = record.timestamp();
        Optional<Integer> leaderEpoch = maybeLeaderEpoch(batch.partitionLeaderEpoch());
        TimestampType timestampType = batch.timestampType();
        Headers headers = new RecordHeaders(record.headers());
        ByteBuffer keyBytes = record.key();
        byte[] keyByteArray = keyBytes == null ? null : Utils.toArray(keyBytes);
        K key = keyBytes == null ? null : this.keyDeserializer.deserialize(partition.topic(), headers, keyByteArray);
        ByteBuffer valueBytes = record.value();
        byte[] valueByteArray = valueBytes == null ? null : Utils.toArray(valueBytes);
        V value = valueBytes == null ? null : this.valueDeserializer.deserialize(partition.topic(), headers, valueByteArray);
        return new ConsumerRecord<>(partition.topic(), partition.partition(), offset, timestamp, timestampType, keyByteArray == null ? ConsumerRecord.NULL_SIZE : keyByteArray.length, valueByteArray == null ? ConsumerRecord.NULL_SIZE : valueByteArray.length, key, value, headers, leaderEpoch);
    } catch (RuntimeException e) {
        throw new RecordDeserializationException(partition, record.offset(), "Error deserializing key/value for partition " + partition + " at offset " + record.offset() + ". If needed, please seek past the record to continue consumption.", e);
    }
}

Example 80 with ConsumerRecord

use of org.apache.kafka.clients.consumer.ConsumerRecord in project kafka by apache.

the class KafkaStatusBackingStore method configure.

@Override
public void configure(final WorkerConfig config) {
    this.statusTopic = config.getString(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG);
    if (this.statusTopic == null || this.statusTopic.trim().length() == 0)
        throw new ConfigException("Must specify topic for connector status.");
    String clusterId = ConnectUtils.lookupKafkaClusterId(config);
    Map<String, Object> originals = config.originals();
    Map<String, Object> producerProps = new HashMap<>(originals);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    // we handle retries in this class
    producerProps.put(ProducerConfig.RETRIES_CONFIG, 0);
    // disable idempotence since retries is force to 0
    producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false);
    ConnectUtils.addMetricsContextProperties(producerProps, config, clusterId);
    Map<String, Object> consumerProps = new HashMap<>(originals);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
    Map<String, Object> adminProps = new HashMap<>(originals);
    ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
    Supplier<TopicAdmin> adminSupplier;
    if (topicAdminSupplier != null) {
        adminSupplier = topicAdminSupplier;
    } else {
        // Create our own topic admin supplier that we'll close when we're stopped
        ownTopicAdmin = new SharedTopicAdmin(adminProps);
        adminSupplier = ownTopicAdmin;
    }
    Map<String, Object> topicSettings = config instanceof DistributedConfig ? ((DistributedConfig) config).statusStorageTopicSettings() : Collections.emptyMap();
    NewTopic topicDescription = TopicAdmin.defineTopic(statusTopic).config(// first so that we override user-supplied settings as needed
    topicSettings).compacted().partitions(config.getInt(DistributedConfig.STATUS_STORAGE_PARTITIONS_CONFIG)).replicationFactor(config.getShort(DistributedConfig.STATUS_STORAGE_REPLICATION_FACTOR_CONFIG)).build();
    Callback<ConsumerRecord<String, byte[]>> readCallback = (error, record) -> read(record);
    this.kafkaLog = createKafkaBasedLog(statusTopic, producerProps, consumerProps, readCallback, topicDescription, adminSupplier);
}