Examples with Repartitioned org.apache.kafka.streams.kstream.Repartitioned used on opensource projects

Example 6 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector.

@Test
public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector() throws Exception {
    final String repartitionedName = "new-key";
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<String, String> repartitioned = Repartitioned.<String, String>as(repartitionedName).withKeySerde(Serdes.String());
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString(), Named.as(repartitionedName)).repartition(repartitioned).groupByKey().count().toStream().to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new StringDeserializer(), new LongDeserializer(), Arrays.asList(new KeyValue<>("1", 1L), new KeyValue<>("2", 1L)));
    final String topology = builder.build().describe().toString();
    final String repartitionTopicName = toRepartitionTopicName(repartitionedName);
    assertTrue(topicExists(repartitionTopicName));
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*" + repartitionedName + "-repartition.*"));
    assertEquals(1, countOccurrencesInTopology(topology, "<-- " + repartitionedName + "\n"));
}

Example 7 with Repartitioned

use of org.apache.kafka.streams.kstream.Repartitioned in project kafka by apache.

the class KStreamRepartitionTest method shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsDoNotMatchWhenJoining.

@Test
public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsDoNotMatchWhenJoining() {
    final String topicB = "topic-b";
    final String outputTopic = "topic-output";
    final String topicBRepartitionedName = "topic-b-scale-up";
    final String inputTopicRepartitionedName = "input-topic-scale-up";
    final int topicBNumberOfPartitions = 2;
    final int inputTopicNumberOfPartitions = 4;
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionedName).withNumberOfPartitions(inputTopicNumberOfPartitions);
    final Repartitioned<Integer, String> topicBRepartitioned = Repartitioned.<Integer, String>as(topicBRepartitionedName).withNumberOfPartitions(topicBNumberOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(topicBRepartitioned);
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    final Map<String, Integer> repartitionTopicsWithNumOfPartitions = Utils.mkMap(Utils.mkEntry(toRepartitionTopicName(topicBRepartitionedName), topicBNumberOfPartitions), Utils.mkEntry(toRepartitionTopicName(inputTopicRepartitionedName), inputTopicNumberOfPartitions));
    final TopologyException expected = assertThrows(TopologyException.class, () -> builder.build(props));
    final String expectedErrorMessage = String.format("Following topics do not have the same " + "number of partitions: [%s]", new TreeMap<>(repartitionTopicsWithNumOfPartitions));
    assertNotNull(expected);
    assertTrue(expected.getMessage().contains(expectedErrorMessage));
}