Examples with KStream org.apache.kafka.streams.kstream.KStream used on opensource projects

Example 61 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining.

@Test
public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException {
    final int topicBNumberOfPartitions = 6;
    final String inputTopicRepartitionName = "join-repartition-test";
    final AtomicReference<Throwable> expectedThrowable = new AtomicReference<>();
    final int inputTopicRepartitionedNumOfPartitions = 2;
    CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionName).withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String()));
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    builder.build(streamsConfiguration);
    startStreams(builder, REBALANCING, ERROR, (t, e) -> expectedThrowable.set(e));
    final String expectedMsg = String.format("Number of partitions [%s] of repartition topic [%s] " + "doesn't match number of partitions [%s] of the source topic.", inputTopicRepartitionedNumOfPartitions, toRepartitionTopicName(inputTopicRepartitionName), topicBNumberOfPartitions);
    assertNotNull(expectedThrowable.get());
    assertTrue(expectedThrowable.get().getMessage().contains(expectedMsg));
}

Example 62 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class InternalTopicIntegrationTest method shouldCompactAndDeleteTopicsForWindowStoreChangelogs.

@Test
public void shouldCompactAndDeleteTopicsForWindowStoreChangelogs() {
    final String appID = APP_ID + "-compact-delete";
    streamsProp.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
    // 
    // Step 1: Configure and start a simple word count topology
    // 
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
    final int durationMs = 2000;
    textLines.flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"))).groupBy(MockMapper.selectValueMapper()).windowedBy(TimeWindows.of(ofSeconds(1L)).grace(ofMillis(0L))).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as("CountWindows").withRetention(ofSeconds(2L)));
    final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProp);
    streams.start();
    // 
    // Step 2: Produce some input data to the input topic.
    // 
    produceData(Arrays.asList("hello", "world", "world", "hello world"));
    // 
    // Step 3: Verify the state changelog topics are compact
    // 
    waitForCompletion(streams, 2, 30000L);
    streams.close();
    final Properties properties = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "CountWindows", null));
    final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
    assertEquals(2, policies.size());
    assertTrue(policies.contains(LogConfig.Compact()));
    assertTrue(policies.contains(LogConfig.Delete()));
    // retention should be 1 day + the window duration
    final long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
    assertEquals(retention, Long.parseLong(properties.getProperty(LogConfig.RetentionMsProp())));
    final Properties repartitionProps = getTopicProperties(appID + "-CountWindows-repartition");
    assertEquals(LogConfig.Delete(), repartitionProps.getProperty(LogConfig.CleanupPolicyProp()));
    assertEquals(4, repartitionProps.size());
}

Example 63 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class JoinWithIncompleteMetadataIntegrationTest method testShouldAutoShutdownOnJoinWithIncompleteMetadata.

@Test
public void testShouldAutoShutdownOnJoinWithIncompleteMetadata() throws InterruptedException {
    STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, APP_ID);
    STREAMS_CONFIG.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    final KStream<Long, String> notExistStream = builder.stream(NON_EXISTENT_INPUT_TOPIC_LEFT);
    final KTable<Long, String> aggregatedTable = notExistStream.leftJoin(rightTable, valueJoiner).groupBy((key, value) -> key).reduce((value1, value2) -> value1 + value2);
    // Write the (continuously updating) results to the output topic.
    aggregatedTable.toStream().to(OUTPUT_TOPIC);
    final KafkaStreamsWrapper streams = new KafkaStreamsWrapper(builder.build(), STREAMS_CONFIG);
    final IntegrationTestUtils.StateListenerStub listener = new IntegrationTestUtils.StateListenerStub();
    streams.setStreamThreadStateListener(listener);
    streams.start();
    TestUtils.waitForCondition(listener::transitToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
    streams.close();
    assertTrue(listener.transitToPendingShutdownSeen());
}

Example 64 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class AbstractResetIntegrationTest method setupTopologyWithIntermediateTopic.

@SuppressWarnings("deprecation")
private Topology setupTopologyWithIntermediateTopic(final boolean useRepartitioned, final String outputTopic2) {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map(KeyValue::new).groupByKey().count().toStream().to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long()));
    final KStream<Long, String> stream;
    if (useRepartitioned) {
        stream = input.repartition();
    } else {
        input.to(INTERMEDIATE_USER_TOPIC);
        stream = builder.stream(INTERMEDIATE_USER_TOPIC);
    }
    stream.groupByKey().windowedBy(TimeWindows.of(ofMillis(35)).advanceBy(ofMillis(10))).count().toStream().map((key, value) -> new KeyValue<>(key.window().start() + key.window().end(), value)).to(outputTopic2, Produced.with(Serdes.Long(), Serdes.Long()));
    return builder.build();
}

Example 65 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class AbstractResetIntegrationTest method setupTopologyWithoutIntermediateUserTopic.

protected Topology setupTopologyWithoutIntermediateUserTopic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map((key, value) -> new KeyValue<>(key, key)).to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long()));
    return builder.build();
}