Examples with IntegerDeserializer org.apache.kafka.common.serialization.IntegerDeserializer used on opensource projects

Example 21 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldUseStreamPartitionerForRepartitionOperation.

@Test
public void shouldUseStreamPartitionerForRepartitionOperation() throws Exception {
    final int partition = 1;
    final String repartitionName = "partitioner-test";
    final long timestamp = System.currentTimeMillis();
    final AtomicInteger partitionerInvocation = new AtomicInteger(0);
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B"));
    sendEvents(timestamp, expectedRecords);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> repartitioned = Repartitioned.<Integer, String>as(repartitionName).withStreamPartitioner((topic, key, value, numPartitions) -> {
        partitionerInvocation.incrementAndGet();
        return partition;
    });
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(repartitioned).to(outputTopic);
    startStreams(builder);
    final String topic = toRepartitionTopicName(repartitionName);
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), expectedRecords);
    assertTrue(topicExists(topic));
    assertEquals(expectedRecords.size(), partitionerInvocation.get());
}

Example 22 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldPerformSelectKeyWithRepartitionOperation.

@Test
public void shouldPerformSelectKeyWithRepartitionOperation() throws Exception {
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "10"), new KeyValue<>(2, "20")));
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> Integer.valueOf(value)).repartition().to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), Arrays.asList(new KeyValue<>(10, "10"), new KeyValue<>(20, "20")));
    final String topology = builder.build().describe().toString();
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition.*"));
}

Example 23 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation.

@Test
public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws Exception {
    final String topicBRepartitionedName = "topic-b-scale-up";
    final String inputTopicRepartitionedName = "input-topic-scale-up";
    final long timestamp = System.currentTimeMillis();
    CLUSTER.createTopic(topicB, 1, 1);
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B"));
    sendEvents(timestamp, expectedRecords);
    sendEvents(topicB, timestamp, expectedRecords);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionedName).withNumberOfPartitions(4);
    final Repartitioned<Integer, String> topicBRepartitioned = Repartitioned.<Integer, String>as(topicBRepartitionedName).withNumberOfPartitions(4);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(topicBRepartitioned);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    startStreams(builder);
    assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBRepartitionedName)));
    assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionedName)));
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), expectedRecords);
}

Example 24 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed.

@Test
public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws Exception {
    final String repartitionName = "dummy";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(Repartitioned.as(repartitionName)).to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final String topology = builder.build().describe().toString();
    assertTrue(topicExists(toRepartitionTopicName(repartitionName)));
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*dummy-repartition.*"));
}

Example 25 with IntegerDeserializer

use of org.apache.kafka.common.serialization.IntegerDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNumberOfPartitionsIsNotSpecified.

@Test
public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNumberOfPartitionsIsNotSpecified() throws Exception {
    final String repartitionName = "new-topic";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(Repartitioned.as(repartitionName)).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new IntegerDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>(1, 1L), new KeyValue<>(2, 1L)));
    final String repartitionTopicName = toRepartitionTopicName(repartitionName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(4, getNumberOfPartitionsForTopic(repartitionTopicName));
}