Examples with StringDeserializer org.apache.kafka.common.serialization.StringDeserializer used on opensource projects

Example 96 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class EosTestDriver method verifySum.

private static void verifySum(final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> inputPerTopicPerPartition, final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> minPerTopicPerPartition) {
    final StringDeserializer stringDeserializer = new StringDeserializer();
    final IntegerDeserializer integerDeserializer = new IntegerDeserializer();
    final LongDeserializer longDeserializer = new LongDeserializer();
    final HashMap<String, Long> currentSumPerKey = new HashMap<>();
    for (final Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords : minPerTopicPerPartition.entrySet()) {
        final TopicPartition inputTopicPartition = new TopicPartition("data", partitionRecords.getKey().partition());
        final List<ConsumerRecord<byte[], byte[]>> partitionInput = inputPerTopicPerPartition.get(inputTopicPartition);
        final List<ConsumerRecord<byte[], byte[]>> partitionSum = partitionRecords.getValue();
        if (partitionInput.size() != partitionSum.size()) {
            throw new RuntimeException("Result verification failed: expected " + partitionInput.size() + " records for " + partitionRecords.getKey() + " but received " + partitionSum.size());
        }
        final Iterator<ConsumerRecord<byte[], byte[]>> inputRecords = partitionInput.iterator();
        for (final ConsumerRecord<byte[], byte[]> receivedRecord : partitionSum) {
            final ConsumerRecord<byte[], byte[]> input = inputRecords.next();
            final String receivedKey = stringDeserializer.deserialize(receivedRecord.topic(), receivedRecord.key());
            final long receivedValue = longDeserializer.deserialize(receivedRecord.topic(), receivedRecord.value());
            final String key = stringDeserializer.deserialize(input.topic(), input.key());
            final int value = integerDeserializer.deserialize(input.topic(), input.value());
            Long sum = currentSumPerKey.get(key);
            if (sum == null) {
                sum = (long) value;
            } else {
                sum += value;
            }
            currentSumPerKey.put(key, sum);
            if (!receivedKey.equals(key) || receivedValue != sum) {
                throw new RuntimeException("Result verification failed for " + receivedRecord + " expected <" + key + "," + sum + "> but was <" + receivedKey + "," + receivedValue + ">");
            }
        }
    }
}

Example 97 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class KStreamAggregationDedupIntegrationTest method shouldReduce.

@Test
public void shouldReduce() throws Exception {
    produceMessages(System.currentTimeMillis());
    groupedStream.reduce(reducer, "reduce-by-key").to(Serdes.String(), Serdes.String(), outputTopic);
    startStreams();
    produceMessages(System.currentTimeMillis());
    List<KeyValue<String, String>> results = receiveMessages(new StringDeserializer(), new StringDeserializer(), 5);
    Collections.sort(results, new Comparator<KeyValue<String, String>>() {

        @Override
        public int compare(KeyValue<String, String> o1, KeyValue<String, String> o2) {
            return KStreamAggregationDedupIntegrationTest.compare(o1, o2);
        }
    });
    assertThat(results, is(Arrays.asList(KeyValue.pair("A", "A:A"), KeyValue.pair("B", "B:B"), KeyValue.pair("C", "C:C"), KeyValue.pair("D", "D:D"), KeyValue.pair("E", "E:E"))));
}

Example 98 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class KStreamAggregationIntegrationTest method shouldCountHelper.

private void shouldCountHelper() throws Exception {
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValue<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 10);
    Collections.sort(results, new Comparator<KeyValue<String, Long>>() {

        @Override
        public int compare(final KeyValue<String, Long> o1, final KeyValue<String, Long> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    });
    assertThat(results, is(Arrays.asList(KeyValue.pair("A", 1L), KeyValue.pair("A", 2L), KeyValue.pair("B", 1L), KeyValue.pair("B", 2L), KeyValue.pair("C", 1L), KeyValue.pair("C", 2L), KeyValue.pair("D", 1L), KeyValue.pair("D", 2L), KeyValue.pair("E", 1L), KeyValue.pair("E", 2L))));
}

Example 99 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class KStreamAggregationIntegrationTest method shouldAggregate.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregate() throws Exception {
    produceMessages(mockTime.milliseconds());
    groupedStream.aggregate(initializer, aggregator, Serdes.Integer(), "aggregate-by-selected-key").to(Serdes.String(), Serdes.Integer(), outputTopic);
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValue<String, Integer>> results = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 10);
    Collections.sort(results, new Comparator<KeyValue<String, Integer>>() {

        @Override
        public int compare(final KeyValue<String, Integer> o1, final KeyValue<String, Integer> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    });
    assertThat(results, is(Arrays.asList(KeyValue.pair("A", 1), KeyValue.pair("A", 2), KeyValue.pair("B", 1), KeyValue.pair("B", 2), KeyValue.pair("C", 1), KeyValue.pair("C", 2), KeyValue.pair("D", 1), KeyValue.pair("D", 2), KeyValue.pair("E", 1), KeyValue.pair("E", 2))));
}

Example 100 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project core-ng-project by neowu.

the class Kafka method consumer.

public Consumer<String, byte[]> consumer(String group, Set<String> topics) {
    if (uri == null)
        throw new Error("uri must not be null");
    StopWatch watch = new StopWatch();
    try {
        Map<String, Object> config = Maps.newHashMap();
        config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, uri);
        config.put(ConsumerConfig.GROUP_ID_CONFIG, group);
        config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        config.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, (int) maxProcessTime.toMillis());
        config.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, (int) maxProcessTime.plusSeconds(5).toMillis());
        config.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
        config.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, maxPollBytes);
        config.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, minPollBytes);
        config.put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, (int) minPollMaxWaitTime.toMillis());
        Consumer<String, byte[]> consumer = new KafkaConsumer<>(config, new StringDeserializer(), new ByteArrayDeserializer());
        consumer.subscribe(topics);
        consumerMetrics.add(consumer.metrics());
        return consumer;
    } finally {
        logger.info("create kafka consumer, uri={}, name={}, topics={}, elapsedTime={}", uri, name, topics, watch.elapsedTime());
    }
}