Examples with Repartitioned org.apache.kafka.streams.kstream.Repartitioned used on opensource projects

Example 1 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupByKey.

@Test
public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupByKey() throws Exception {
    final String repartitionName = "new-partitions";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionName).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()).withNumberOfPartitions(1);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString()).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    final String topology = builder.build().describe().toString();
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    assertTrue(topicExists(toRepartitionTopicName(repartitionName)));
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition"));
}

Example 2 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldDeductNumberOfPartitionsFromRepartitionOperation.

@Test
public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws Exception {
    final String topicBMapperName = "topic-b-mapper";
    final int topicBNumberOfPartitions = 6;
    final String inputTopicRepartitionName = "join-repartition-test";
    final int inputTopicRepartitionedNumOfPartitions = 3;
    final long timestamp = System.currentTimeMillis();
    CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1);
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B"));
    sendEvents(timestamp, expectedRecords);
    sendEvents(topicB, timestamp, expectedRecords);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionName).withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())).map(KeyValue::new, Named.as(topicBMapperName));
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    builder.build(streamsConfiguration);
    startStreams(builder);
    assertEquals(inputTopicRepartitionedNumOfPartitions, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionName)));
    assertEquals(inputTopicRepartitionedNumOfPartitions, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBMapperName)));
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), expectedRecords);
}

Example 3 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldGoThroughRebalancingCorrectly.

@Test
public void shouldGoThroughRebalancingCorrectly() throws Exception {
    final String repartitionName = "rebalancing-test";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionName).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()).withNumberOfPartitions(2);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString()).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    final Properties streamsToCloseConfigs = new Properties();
    streamsToCloseConfigs.putAll(streamsConfiguration);
    streamsToCloseConfigs.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath() + "-2");
    final KafkaStreams kafkaStreamsToClose = startStreams(builder, streamsToCloseConfigs);
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    kafkaStreamsToClose.close();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "C"), new KeyValue<>(2, "D")));
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 2L), new KeyValue<>("2", 2L)));
    final String repartitionTopicName = toRepartitionTopicName(repartitionName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName));
}

Example 4 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation.

@Test
public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws Exception {
    final String topicBRepartitionedName = "topic-b-scale-up";
    final String inputTopicRepartitionedName = "input-topic-scale-up";
    final long timestamp = System.currentTimeMillis();
    CLUSTER.createTopic(topicB, 1, 1);
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B"));
    sendEvents(timestamp, expectedRecords);
    sendEvents(topicB, timestamp, expectedRecords);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionedName).withNumberOfPartitions(4);
    final Repartitioned<Integer, String> topicBRepartitioned = Repartitioned.<Integer, String>as(topicBRepartitionedName).withNumberOfPartitions(4);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(topicBRepartitioned);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    startStreams(builder);
    assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBRepartitionedName)));
    assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionedName)));
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), expectedRecords);
}

Example 5 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining.

@Test
public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException {
    final int topicBNumberOfPartitions = 6;
    final String inputTopicRepartitionName = "join-repartition-test";
    final AtomicReference<Throwable> expectedThrowable = new AtomicReference<>();
    final int inputTopicRepartitionedNumOfPartitions = 2;
    CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionName).withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String()));
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    builder.build(streamsConfiguration);
    startStreams(builder, REBALANCING, ERROR, (t, e) -> expectedThrowable.set(e));
    final String expectedMsg = String.format("Number of partitions [%s] of repartition topic [%s] " + "doesn't match number of partitions [%s] of the source topic.", inputTopicRepartitionedNumOfPartitions, toRepartitionTopicName(inputTopicRepartitionName), topicBNumberOfPartitions);
    assertNotNull(expectedThrowable.get());
    assertTrue(expectedThrowable.get().getMessage().contains(expectedMsg));
}