Examples with IntegerDeserializer org.apache.kafka.common.serialization.IntegerDeserializer used on opensource projects

Example 16 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class EosTestDriver method verifyReceivedAllRecords.

private static void verifyReceivedAllRecords(final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> expectedRecords, final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> receivedRecords) {
    if (expectedRecords.size() != receivedRecords.size()) {
        throw new RuntimeException("Result verification failed. Received " + receivedRecords.size() + " records but expected " + expectedRecords.size());
    }
    final StringDeserializer stringDeserializer = new StringDeserializer();
    final IntegerDeserializer integerDeserializer = new IntegerDeserializer();
    for (final Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords : receivedRecords.entrySet()) {
        final TopicPartition inputTopicPartition = new TopicPartition("data", partitionRecords.getKey().partition());
        final Iterator<ConsumerRecord<byte[], byte[]>> expectedRecord = expectedRecords.get(inputTopicPartition).iterator();
        for (final ConsumerRecord<byte[], byte[]> receivedRecord : partitionRecords.getValue()) {
            final ConsumerRecord<byte[], byte[]> expected = expectedRecord.next();
            final String receivedKey = stringDeserializer.deserialize(receivedRecord.topic(), receivedRecord.key());
            final int receivedValue = integerDeserializer.deserialize(receivedRecord.topic(), receivedRecord.value());
            final String expectedKey = stringDeserializer.deserialize(expected.topic(), expected.key());
            final int expectedValue = integerDeserializer.deserialize(expected.topic(), expected.value());
            if (!receivedKey.equals(expectedKey) || receivedValue != expectedValue) {
                throw new RuntimeException("Result verification failed for " + receivedRecord + " expected <" + expectedKey + "," + expectedValue + "> but was <" + receivedKey + "," + receivedValue + ">");
            }
        }
    }
}

Example 17 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class EosTestDriver method verifySum.

private static void verifySum(final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> inputPerTopicPerPartition, final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> minPerTopicPerPartition) {
    final StringDeserializer stringDeserializer = new StringDeserializer();
    final IntegerDeserializer integerDeserializer = new IntegerDeserializer();
    final LongDeserializer longDeserializer = new LongDeserializer();
    final HashMap<String, Long> currentSumPerKey = new HashMap<>();
    for (final Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords : minPerTopicPerPartition.entrySet()) {
        final TopicPartition inputTopicPartition = new TopicPartition("data", partitionRecords.getKey().partition());
        final List<ConsumerRecord<byte[], byte[]>> partitionInput = inputPerTopicPerPartition.get(inputTopicPartition);
        final List<ConsumerRecord<byte[], byte[]>> partitionSum = partitionRecords.getValue();
        if (partitionInput.size() != partitionSum.size()) {
            throw new RuntimeException("Result verification failed: expected " + partitionInput.size() + " records for " + partitionRecords.getKey() + " but received " + partitionSum.size());
        }
        final Iterator<ConsumerRecord<byte[], byte[]>> inputRecords = partitionInput.iterator();
        for (final ConsumerRecord<byte[], byte[]> receivedRecord : partitionSum) {
            final ConsumerRecord<byte[], byte[]> input = inputRecords.next();
            final String receivedKey = stringDeserializer.deserialize(receivedRecord.topic(), receivedRecord.key());
            final long receivedValue = longDeserializer.deserialize(receivedRecord.topic(), receivedRecord.value());
            final String key = stringDeserializer.deserialize(input.topic(), input.key());
            final int value = integerDeserializer.deserialize(input.topic(), input.value());
            Long sum = currentSumPerKey.get(key);
            if (sum == null) {
                sum = (long) value;
            } else {
                sum += value;
            }
            currentSumPerKey.put(key, sum);
            if (!receivedKey.equals(key) || receivedValue != sum) {
                throw new RuntimeException("Result verification failed for " + receivedRecord + " expected <" + key + "," + sum + "> but was <" + receivedKey + "," + receivedValue + ">");
            }
        }
    }
}

Example 18 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project apache-kafka-on-k8s by banzaicloud.

the class KStreamAggregationIntegrationTest method shouldAggregate.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregate() throws Exception {
    produceMessages(mockTime.milliseconds());
    groupedStream.aggregate(initializer, aggregator, Serdes.Integer(), "aggregate-by-selected-key").to(Serdes.String(), Serdes.Integer(), outputTopic);
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValue<String, Integer>> results = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 10);
    Collections.sort(results, new Comparator<KeyValue<String, Integer>>() {

        @Override
        public int compare(final KeyValue<String, Integer> o1, final KeyValue<String, Integer> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    });
    assertThat(results, is(Arrays.asList(KeyValue.pair("A", 1), KeyValue.pair("A", 2), KeyValue.pair("B", 1), KeyValue.pair("B", 2), KeyValue.pair("C", 1), KeyValue.pair("C", 2), KeyValue.pair("D", 1), KeyValue.pair("D", 2), KeyValue.pair("E", 1), KeyValue.pair("E", 2))));
}

Example 19 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateSlidingWindows.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregateSlidingWindows() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    final long timeDifference = 500L;
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = firstBatchTimestamp + timeDifference / 2;
    produceMessages(secondBatchTimestamp);
    final long thirdBatchTimestamp = firstBatchTimestamp + timeDifference - 100L;
    produceMessages(thirdBatchTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, timeDifference);
    // noinspection deprecation
    groupedStream.windowedBy(SlidingWindows.withTimeDifferenceAndGrace(ofMillis(500L), ofMinutes(5))).aggregate(initializer, aggregator, Materialized.with(null, Serdes.Integer())).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.Integer()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, Integer>> windowedMessages = receiveMessagesWithTimestamp(new TimeWindowedDeserializer<>(), new IntegerDeserializer(), String.class, 30);
    // read from ConsoleConsumer
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new IntegerDeserializer(), String.class, 30, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, Integer>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, Integer> o) -> o.key().key()).thenComparingInt(KeyValueTimestamp::value);
    windowedMessages.sort(comparator);
    final long firstBatchLeftWindowStart = firstBatchTimestamp - timeDifference;
    final long firstBatchLeftWindowEnd = firstBatchLeftWindowStart + timeDifference;
    final long firstBatchRightWindowStart = firstBatchTimestamp + 1;
    final long firstBatchRightWindowEnd = firstBatchRightWindowStart + timeDifference;
    final long secondBatchLeftWindowStart = secondBatchTimestamp - timeDifference;
    final long secondBatchLeftWindowEnd = secondBatchLeftWindowStart + timeDifference;
    final long secondBatchRightWindowStart = secondBatchTimestamp + 1;
    final long secondBatchRightWindowEnd = secondBatchRightWindowStart + timeDifference;
    final long thirdBatchLeftWindowStart = thirdBatchTimestamp - timeDifference;
    final long thirdBatchLeftWindowEnd = thirdBatchLeftWindowStart + timeDifference;
    final List<KeyValueTimestamp<Windowed<String>, Integer>> expectResult = Arrays.asList(// A @ firstBatchTimestamp left window created when A @ firstBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), // A @ firstBatchTimestamp right window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), // A @ secondBatchTimestamp right window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), // A @ secondBatchTimestamp left window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), // A @ firstBatchTimestamp right window updated when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), // A @ thirdBatchTimestamp left window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp));
    assertThat(windowedMessages, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, Integer> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}

Example 20 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateWindowed.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregateWindowed() throws Exception {
    final long firstTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstTimestamp);
    final long secondTimestamp = mockTime.milliseconds();
    produceMessages(secondTimestamp);
    produceMessages(secondTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, 500L);
    // noinspection deprecation
    groupedStream.windowedBy(TimeWindows.of(ofMillis(500L))).aggregate(initializer, aggregator, Materialized.with(null, Serdes.Integer())).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.Integer()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, Integer>> windowedMessages = receiveMessagesWithTimestamp(new TimeWindowedDeserializer<>(new StringDeserializer(), 500L), new IntegerDeserializer(), String.class, 15);
    // read from ConsoleConsumer
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new IntegerDeserializer(), String.class, 15, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, Integer>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, Integer> o) -> o.key().key()).thenComparingInt(KeyValueTimestamp::value);
    windowedMessages.sort(comparator);
    final long firstWindowStart = firstTimestamp / 500 * 500;
    final long firstWindowEnd = firstWindowStart + 500;
    final long secondWindowStart = secondTimestamp / 500 * 500;
    final long secondWindowEnd = secondWindowStart + 500;
    final List<KeyValueTimestamp<Windowed<String>, Integer>> expectResult = Arrays.asList(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp));
    assertThat(windowedMessages, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, Integer> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}