Examples with ByteArrayDeserializer org.apache.kafka.common.serialization.ByteArrayDeserializer used on opensource projects

Example 11 with ByteArrayDeserializer

use of org.apache.kafka.common.serialization.ByteArrayDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testReadCommittedWithCommittedAndAbortedTransactions.

@Test
public void testReadCommittedWithCommittedAndAbortedTransactions() {
    Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
    long pid1 = 1L;
    long pid2 = 2L;
    // Appends for producer 1 (eventually committed)
    appendTransactionalRecords(buffer, pid1, 0L, new SimpleRecord("commit1-1".getBytes(), "value".getBytes()), new SimpleRecord("commit1-2".getBytes(), "value".getBytes()));
    // Appends for producer 2 (eventually aborted)
    appendTransactionalRecords(buffer, pid2, 2L, new SimpleRecord("abort2-1".getBytes(), "value".getBytes()));
    // commit producer 1
    commitTransaction(buffer, pid1, 3L);
    // append more for producer 2 (eventually aborted)
    appendTransactionalRecords(buffer, pid2, 4L, new SimpleRecord("abort2-2".getBytes(), "value".getBytes()));
    // abort producer 2
    abortTransaction(buffer, pid2, 5L);
    abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 2L));
    // New transaction for producer 1 (eventually aborted)
    appendTransactionalRecords(buffer, pid1, 6L, new SimpleRecord("abort1-1".getBytes(), "value".getBytes()));
    // New transaction for producer 2 (eventually committed)
    appendTransactionalRecords(buffer, pid2, 7L, new SimpleRecord("commit2-1".getBytes(), "value".getBytes()));
    // Add messages for producer 1 (eventually aborted)
    appendTransactionalRecords(buffer, pid1, 8L, new SimpleRecord("abort1-2".getBytes(), "value".getBytes()));
    // abort producer 1
    abortTransaction(buffer, pid1, 9L);
    abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 6));
    // commit producer 2
    commitTransaction(buffer, pid2, 10L);
    buffer.flip();
    MemoryRecords records = MemoryRecords.readableRecords(buffer);
    subscriptions.assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    // normal fetch
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
    consumerClient.poll(0);
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
    assertTrue(fetchedRecords.containsKey(tp0));
    // There are only 3 committed records
    List<ConsumerRecord<byte[], byte[]>> fetchedConsumerRecords = fetchedRecords.get(tp0);
    Set<String> fetchedKeys = new HashSet<>();
    for (ConsumerRecord<byte[], byte[]> consumerRecord : fetchedConsumerRecords) {
        fetchedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8));
    }
    assertEquals(Utils.mkSet("commit1-1", "commit1-2", "commit2-1"), fetchedKeys);
}

Example 12 with ByteArrayDeserializer

use of org.apache.kafka.common.serialization.ByteArrayDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testReadCommittedLagMetric.

@Test
public void testReadCommittedLagMetric() {
    Metrics metrics = new Metrics();
    fetcher = createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
    subscriptions.assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax);
    Map<String, String> tags = new HashMap<>();
    tags.put("topic", tp0.topic());
    tags.put("partition", String.valueOf(tp0.partition()));
    MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags);
    MetricName partitionLagMetricDeprecated = metrics.metricName(tp0 + ".records-lag", metricGroup);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric);
    // recordsFetchLagMax should be initialized to negative infinity
    assertEquals(Double.NEGATIVE_INFINITY, recordsFetchLagMax.value(), EPSILON);
    // recordsFetchLagMax should be lso - fetchOffset after receiving an empty FetchResponse
    fetchRecords(tp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 50L, 0);
    assertEquals(50, recordsFetchLagMax.value(), EPSILON);
    KafkaMetric partitionLag = allMetrics.get(partitionLagMetric);
    assertEquals(50, partitionLag.value(), EPSILON);
    KafkaMetric partitionLagDeprecated = allMetrics.get(partitionLagMetricDeprecated);
    assertEquals(50, partitionLagDeprecated.value(), EPSILON);
    // recordsFetchLagMax should be lso - offset of the last message after receiving a non-empty FetchResponse
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    fetchRecords(tp0, builder.build(), Errors.NONE, 200L, 150L, 0);
    assertEquals(147, recordsFetchLagMax.value(), EPSILON);
    assertEquals(147, partitionLag.value(), EPSILON);
    // verify de-registration of partition lag
    subscriptions.unsubscribe();
    assertFalse(allMetrics.containsKey(partitionLagMetric));
    assertFalse(allMetrics.containsKey(partitionLagMetricDeprecated));
}

Example 13 with ByteArrayDeserializer

use of org.apache.kafka.common.serialization.ByteArrayDeserializer in project JavaForFun by gumartinm.

the class OffsetManagement method manageOffsets.

private void manageOffsets() {
    final String inputTopic = options.valueOf(inputTopicOption);
    if (!isAvailable(inputTopic)) {
        System.err.println("Chosen topic does not exist: " + inputTopic);
        return;
    }
    final Properties config = new Properties();
    config.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, options.valueOf(bootstrapServerOption));
    config.setProperty(ConsumerConfig.GROUP_ID_CONFIG, options.valueOf(groupIdOption));
    config.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    try (final KafkaConsumer<byte[], byte[]> client = new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())) {
        Collection<String> topicsToSubscribe = new ArrayList<>();
        topicsToSubscribe.add(inputTopic);
        client.subscribe(topicsToSubscribe);
        client.poll(1);
        final Collection<TopicPartition> partitions = client.assignment();
        if (partitions.isEmpty()) {
            System.err.println("No partitions for input topic: " + inputTopic);
            return;
        }
        final Collection<TopicPartition> filteredTopicPartitions = filterTopicPartition(partitions);
        if (filteredTopicPartitions.isEmpty()) {
            System.err.println("No partitions with the chosen value: " + options.valueOf(partitionOption));
            return;
        }
        Long offset = options.valueOf(offsetOption);
        if (offset.equals(OffsetRequest.EarliestTime())) {
            client.seekToBeginning(filteredTopicPartitions);
            for (final TopicPartition partition : filteredTopicPartitions) {
                client.position(partition);
            }
        } else {
            for (final TopicPartition partition : filteredTopicPartitions) {
                client.seek(partition, offset);
                client.position(partition);
            }
        }
        client.commitSync();
    }
}

Example 14 with ByteArrayDeserializer

use of org.apache.kafka.common.serialization.ByteArrayDeserializer in project drill by apache.

the class KafkaQueriesTest method fetchOffsets.

private Map<TopicPartition, Long> fetchOffsets(int flag) throws InterruptedException {
    Consumer<byte[], byte[]> kafkaConsumer = null;
    try {
        kafkaConsumer = new KafkaConsumer<>(storagePluginConfig.getKafkaConsumerProps(), new ByteArrayDeserializer(), new ByteArrayDeserializer());
        Map<TopicPartition, Long> offsetsMap = new HashMap<>();
        kafkaConsumer.subscribe(Collections.singletonList(TestQueryConstants.JSON_TOPIC));
        // based on KafkaConsumer JavaDoc, seekToBeginning/seekToEnd functions
        // evaluates lazily, seeking to the
        // first/last offset in all partitions only when poll(long) or
        // position(TopicPartition) are called
        kafkaConsumer.poll(Duration.ofSeconds(5));
        Set<TopicPartition> assignments = waitForConsumerAssignment(kafkaConsumer);
        if (flag == -2) {
            // fetch start offsets for each topicPartition
            kafkaConsumer.seekToBeginning(assignments);
            for (TopicPartition topicPartition : assignments) {
                offsetsMap.put(topicPartition, kafkaConsumer.position(topicPartition));
            }
        } else if (flag == -1) {
            // fetch end offsets for each topicPartition
            kafkaConsumer.seekToEnd(assignments);
            for (TopicPartition topicPartition : assignments) {
                offsetsMap.put(topicPartition, kafkaConsumer.position(topicPartition));
            }
        } else {
            throw new RuntimeException(String.format("Unsupported flag %d", flag));
        }
        return offsetsMap;
    } finally {
        embeddedKafkaCluster.registerToClose(kafkaConsumer);
    }
}

Example 15 with ByteArrayDeserializer

use of org.apache.kafka.common.serialization.ByteArrayDeserializer in project samza by apache.

the class KafkaConsumerConfig method getKafkaSystemConsumerConfig.

/**
 * Create kafka consumer configs, based on the subset of global configs.
 * @param config application config
 * @param systemName system name
 * @param clientId client id provided by the caller
 * @return KafkaConsumerConfig
 */
public static KafkaConsumerConfig getKafkaSystemConsumerConfig(Config config, String systemName, String clientId) {
    Config subConf = config.subset(String.format("systems.%s.consumer.", systemName), true);
    final String groupId = createConsumerGroupId(config);
    Map<String, Object> consumerProps = new HashMap<>(subConf);
    consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId);
    // These are values we enforce in sazma, and they cannot be overwritten.
    // Disable consumer auto-commit because Samza controls commits
    consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    // check if samza default offset value is defined
    String systemOffsetDefault = new SystemConfig(config).getSystemOffsetDefault(systemName);
    // Translate samza config value to kafka config value
    String autoOffsetReset = getAutoOffsetResetValue((String) consumerProps.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), systemOffsetDefault);
    LOG.info("setting auto.offset.reset for system {} to {}", systemName, autoOffsetReset);
    consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
    // if consumer bootstrap servers are not configured, get them from the producer configs
    if (!subConf.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
        String bootstrapServers = config.get(String.format("systems.%s.producer.%s", systemName, ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));
        if (StringUtils.isEmpty(bootstrapServers)) {
            throw new SamzaException("Missing " + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " config  for " + systemName);
        }
        consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    }
    // Always use default partition assignment strategy. Do not allow override.
    consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, RangeAssignor.class.getName());
    // default to byte[]
    if (!consumerProps.containsKey(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG)) {
        LOG.info("setting key serialization for the consumer(for system {}) to ByteArrayDeserializer", systemName);
        consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    }
    if (!consumerProps.containsKey(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG)) {
        LOG.info("setting value serialization for the consumer(for system {}) to ByteArrayDeserializer", systemName);
        consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    }
    // Override default max poll config if there is no value
    consumerProps.putIfAbsent(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, DEFAULT_KAFKA_CONSUMER_MAX_POLL_RECORDS);
    return new KafkaConsumerConfig(consumerProps, systemName);
}