Examples with LongDeserializer org.apache.kafka.common.serialization.LongDeserializer used on opensource projects

Example 36 with LongDeserializer

use of org.apache.kafka.common.serialization.LongDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldGoThroughRebalancingCorrectly.

@Test
public void shouldGoThroughRebalancingCorrectly() throws Exception {
    final String repartitionName = "rebalancing-test";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionName).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()).withNumberOfPartitions(2);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString()).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    final Properties streamsToCloseConfigs = new Properties();
    streamsToCloseConfigs.putAll(streamsConfiguration);
    streamsToCloseConfigs.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath() + "-2");
    final KafkaStreams kafkaStreamsToClose = startStreams(builder, streamsToCloseConfigs);
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    kafkaStreamsToClose.close();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "C"), new KeyValue<>(2, "D")));
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 2L), new KeyValue<>("2", 2L)));
    final String repartitionTopicName = toRepartitionTopicName(repartitionName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName));
}

Example 37 with LongDeserializer

use of org.apache.kafka.common.serialization.LongDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector.

@Test
public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector() throws Exception {
    final String repartitionedName = "new-key";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionedName).withKeySerde(Serdes.String());
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString(), Named.as(repartitionedName)).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    final String topology = builder.build().describe().toString();
    final String repartitionTopicName = toRepartitionTopicName(repartitionedName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*" + repartitionedName + "-repartition.*"));
    assertEquals(1, countOccurrencesInTopology(topology, "<-- " + repartitionedName + "\n"));
}

Example 38 with LongDeserializer

use of org.apache.kafka.common.serialization.LongDeserializer in project kafka by apache.

the class EosTestDriver method verifySum.

private static void verifySum(final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> inputPerTopicPerPartition, final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> minPerTopicPerPartition) {
    final StringDeserializer stringDeserializer = new StringDeserializer();
    final IntegerDeserializer integerDeserializer = new IntegerDeserializer();
    final LongDeserializer longDeserializer = new LongDeserializer();
    final HashMap<String, Long> currentSumPerKey = new HashMap<>();
    for (final Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords : minPerTopicPerPartition.entrySet()) {
        final TopicPartition inputTopicPartition = new TopicPartition("data", partitionRecords.getKey().partition());
        final List<ConsumerRecord<byte[], byte[]>> partitionInput = inputPerTopicPerPartition.get(inputTopicPartition);
        final List<ConsumerRecord<byte[], byte[]>> partitionSum = partitionRecords.getValue();
        if (partitionInput.size() != partitionSum.size()) {
            throw new RuntimeException("Result verification failed: expected " + partitionInput.size() + " records for " + partitionRecords.getKey() + " but received " + partitionSum.size());
        }
        final Iterator<ConsumerRecord<byte[], byte[]>> inputRecords = partitionInput.iterator();
        for (final ConsumerRecord<byte[], byte[]> receivedRecord : partitionSum) {
            final ConsumerRecord<byte[], byte[]> input = inputRecords.next();
            final String receivedKey = stringDeserializer.deserialize(receivedRecord.topic(), receivedRecord.key());
            final long receivedValue = longDeserializer.deserialize(receivedRecord.topic(), receivedRecord.value());
            final String key = stringDeserializer.deserialize(input.topic(), input.key());
            final int value = integerDeserializer.deserialize(input.topic(), input.value());
            Long sum = currentSumPerKey.get(key);
            if (sum == null) {
                sum = (long) value;
            } else {
                sum += value;
            }
            currentSumPerKey.put(key, sum);
            if (!receivedKey.equals(key) || receivedValue != sum) {
                throw new RuntimeException("Result verification failed for " + receivedRecord + " expected <" + key + "," + sum + "> but was <" + receivedKey + "," + receivedValue + ">");
            }
        }
    }
}