Examples with REBALANCING org.apache.kafka.streams.KafkaStreams.State.REBALANCING used on opensource projects

Example 1 with REBALANCING

use of org.apache.kafka.streams.KafkaStreams.State.REBALANCING in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldGoThroughRebalancingCorrectly.

@Test
public void shouldGoThroughRebalancingCorrectly() throws Exception {
    final String repartitionName = "rebalancing-test";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionName).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()).withNumberOfPartitions(2);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString()).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    final Properties streamsToCloseConfigs = new Properties();
    streamsToCloseConfigs.putAll(streamsConfiguration);
    streamsToCloseConfigs.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath() + "-2");
    final KafkaStreams kafkaStreamsToClose = startStreams(builder, streamsToCloseConfigs);
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    kafkaStreamsToClose.close();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "C"), new KeyValue<>(2, "D")));
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 2L), new KeyValue<>("2", 2L)));
    final String repartitionTopicName = toRepartitionTopicName(repartitionName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName));
}

Example 2 with REBALANCING

use of org.apache.kafka.streams.KafkaStreams.State.REBALANCING in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining.

@Test
public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException {
    final int topicBNumberOfPartitions = 6;
    final String inputTopicRepartitionName = "join-repartition-test";
    final AtomicReference<Throwable> expectedThrowable = new AtomicReference<>();
    final int inputTopicRepartitionedNumOfPartitions = 2;
    CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionName).withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String()));
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    builder.build(streamsConfiguration);
    startStreams(builder, REBALANCING, ERROR, (t, e) -> expectedThrowable.set(e));
    final String expectedMsg = String.format("Number of partitions [%s] of repartition topic [%s] " + "doesn't match number of partitions [%s] of the source topic.", inputTopicRepartitionedNumOfPartitions, toRepartitionTopicName(inputTopicRepartitionName), topicBNumberOfPartitions);
    assertNotNull(expectedThrowable.get());
    assertTrue(expectedThrowable.get().getMessage().contains(expectedMsg));
}