Examples with KStream org.apache.kafka.streams.kstream.KStream used on opensource projects

Example 71 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class NamedTopologyIntegrationTest method shouldPrefixAllInternalTopicNamesWithNamedTopology.

@Test
public void shouldPrefixAllInternalTopicNamesWithNamedTopology() throws Exception {
    final String countTopologyName = "count-topology";
    final String fkjTopologyName = "FKJ-topology";
    final NamedTopologyBuilder countBuilder = streams.newNamedTopologyBuilder(countTopologyName);
    countBuilder.stream(INPUT_STREAM_1).groupBy((k, v) -> k).count();
    final NamedTopologyBuilder fkjBuilder = streams.newNamedTopologyBuilder(fkjTopologyName);
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final KTable<String, Long> left = fkjBuilder.table(INPUT_STREAM_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    final KTable<String, Long> right = fkjBuilder.table(INPUT_STREAM_3, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    left.join(right, Object::toString, (value1, value2) -> String.valueOf(value1 + value2), Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), props, false)));
    streams.start(asList(fkjBuilder.build(), countBuilder.build()));
    waitForApplicationState(singletonList(streams), State.RUNNING, Duration.ofSeconds(60));
    final String countTopicPrefix = TOPIC_PREFIX + "-" + countTopologyName;
    final String fkjTopicPrefix = TOPIC_PREFIX + "-" + fkjTopologyName;
    final Set<String> internalTopics = CLUSTER.getAllTopicsInCluster().stream().filter(t -> t.contains(TOPIC_PREFIX)).filter(t -> t.endsWith("-repartition") || t.endsWith("-changelog") || t.endsWith("-topic")).collect(Collectors.toSet());
    assertThat(internalTopics, is(mkSet(countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition", countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-changelog", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000010-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_2 + "-STATE-STORE-0000000000-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_3 + "-STATE-STORE-0000000003-changelog")));
}

Example 72 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class NamedTopologyIntegrationTest method shouldAddToEmptyInitialTopologyRemoveResetOffsetsThenAddSameNamedTopology.

@Test
public void shouldAddToEmptyInitialTopologyRemoveResetOffsetsThenAddSameNamedTopology() throws Exception {
    CLUSTER.createTopics(SUM_OUTPUT, COUNT_OUTPUT);
    // Build up named topology with two stateful subtopologies
    final KStream<String, Long> inputStream1 = topology1Builder.stream(INPUT_STREAM_1);
    inputStream1.groupByKey().count().toStream().to(COUNT_OUTPUT);
    inputStream1.groupByKey().reduce(Long::sum).toStream().to(SUM_OUTPUT);
    streams.start();
    final NamedTopology namedTopology = topology1Builder.build();
    streams.addNamedTopology(namedTopology).all().get();
    assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, COUNT_OUTPUT, 3), equalTo(COUNT_OUTPUT_DATA));
    assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, SUM_OUTPUT, 3), equalTo(SUM_OUTPUT_DATA));
    streams.removeNamedTopology("topology-1", true).all().get();
    streams.cleanUpNamedTopology("topology-1");
    CLUSTER.getAllTopicsInCluster().stream().filter(t -> t.contains("changelog")).forEach(t -> {
        try {
            CLUSTER.deleteTopicAndWait(t);
        } catch (final InterruptedException e) {
            e.printStackTrace();
        }
    });
    final KStream<String, Long> inputStream = topology1BuilderDup.stream(INPUT_STREAM_1);
    inputStream.groupByKey().count().toStream().to(COUNT_OUTPUT);
    inputStream.groupByKey().reduce(Long::sum).toStream().to(SUM_OUTPUT);
    final NamedTopology namedTopologyDup = topology1BuilderDup.build();
    streams.addNamedTopology(namedTopologyDup).all().get();
    assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, COUNT_OUTPUT, 3), equalTo(COUNT_OUTPUT_DATA));
    assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, SUM_OUTPUT, 3), equalTo(SUM_OUTPUT_DATA));
    CLUSTER.deleteTopicsAndWait(SUM_OUTPUT, COUNT_OUTPUT);
}

Example 73 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class SuppressionIntegrationTest method shouldAllowDisablingChangelog.

@Test
public void shouldAllowDisablingChangelog() {
    final String testId = "-shouldAllowDisablingChangelog";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull().withLoggingDisabled())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        final Set<String> suppressChangeLog = CLUSTER.getAllTopicsInCluster().stream().filter(s -> s.contains("-changelog")).filter(s -> s.contains("KTABLE-SUPPRESS")).collect(Collectors.toSet());
        assertThat(suppressChangeLog, is(empty()));
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 74 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class KStreamImplTest method shouldSupportForeignKeyTableTableJoinWithKTableFromKStream.

@Test
public void shouldSupportForeignKeyTableTableJoinWithKTableFromKStream() {
    final Consumed<String, String> consumed = Consumed.with(Serdes.String(), Serdes.String());
    final StreamsBuilder builder = new StreamsBuilder();
    final String input1 = "input1";
    final String input2 = "input2";
    final String output = "output";
    final KTable<String, String> leftTable = builder.stream(input1, consumed).toTable();
    final KTable<String, String> rightTable = builder.stream(input2, consumed).toTable();
    final Function<String, String> extractor = value -> value.split("\\|")[1];
    final ValueJoiner<String, String, String> joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
    leftTable.join(rightTable, extractor, joiner).toStream().to(output);
    final Topology topology = builder.build(props);
    final String topologyDescription = topology.describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KTABLE-SOURCE-0000000016 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic])\n" + "      --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000017\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input1])\n" + "      --> KSTREAM-TOTABLE-0000000001\n" + "    Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000017 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000002])\n" + "      --> KTABLE-FK-JOIN-OUTPUT-0000000018\n" + "      <-- KTABLE-SOURCE-0000000016\n" + "    Processor: KSTREAM-TOTABLE-0000000001 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000002])\n" + "      --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000007\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: KTABLE-FK-JOIN-OUTPUT-0000000018 (stores: [])\n" + "      --> KTABLE-TOSTREAM-0000000020\n" + "      <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000017\n" + "    Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000007 (stores: [])\n" + "      --> KTABLE-SINK-0000000008\n" + "      <-- KSTREAM-TOTABLE-0000000001\n" + "    Processor: KTABLE-TOSTREAM-0000000020 (stores: [])\n" + "      --> KSTREAM-SINK-0000000021\n" + "      <-- KTABLE-FK-JOIN-OUTPUT-0000000018\n" + "    Sink: KSTREAM-SINK-0000000021 (topic: output)\n" + "      <-- KTABLE-TOSTREAM-0000000020\n" + "    Sink: KTABLE-SINK-0000000008 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic)\n" + "      <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000007\n" + "\n" + "  Sub-topology: 1\n" + "    Source: KSTREAM-SOURCE-0000000003 (topics: [input2])\n" + "      --> KSTREAM-TOTABLE-0000000004\n" + "    Source: KTABLE-SOURCE-0000000009 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic])\n" + "      --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000011\n" + "    Processor: KSTREAM-TOTABLE-0000000004 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000005])\n" + "      --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000013\n" + "      <-- KSTREAM-SOURCE-0000000003\n" + "    Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000011 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000010])\n" + "      --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000012\n" + "      <-- KTABLE-SOURCE-0000000009\n" + "    Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000012 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000005])\n" + "      --> KTABLE-SINK-0000000015\n" + "      <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000011\n" + "    Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000013 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000010])\n" + "      --> KTABLE-SINK-0000000015\n" + "      <-- KSTREAM-TOTABLE-0000000004\n" + "    Sink: KTABLE-SINK-0000000015 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic)\n" + "      <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000012, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000013\n\n"));
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> left = driver.createInputTopic(input1, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> right = driver.createInputTopic(input2, new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> outputTopic = driver.createOutputTopic(output, new StringDeserializer(), new StringDeserializer());
        // Pre-populate the RHS records. This test is all about what happens when we add/remove LHS records
        right.pipeInput("rhs1", "rhsValue1");
        right.pipeInput("rhs2", "rhsValue2");
        // this unreferenced FK won't show up in any results
        right.pipeInput("rhs3", "rhsValue3");
        assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
        left.pipeInput("lhs1", "lhsValue1|rhs1");
        left.pipeInput("lhs2", "lhsValue2|rhs2");
        final Map<String, String> expected = mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"), mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"));
        assertThat(outputTopic.readKeyValuesToMap(), is(expected));
        // Add another reference to an existing FK
        left.pipeInput("lhs3", "lhsValue3|rhs1");
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))));
        left.pipeInput("lhs1", (String) null);
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", null))));
    }
}

Example 75 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class KStreamImplTest method testNumProcesses.

// specifically testing the deprecated variant
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testNumProcesses() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> source1 = builder.stream(Arrays.asList("topic-1", "topic-2"), stringConsumed);
    final KStream<String, String> source2 = builder.stream(Arrays.asList("topic-3", "topic-4"), stringConsumed);
    final KStream<String, String> stream1 = source1.filter((key, value) -> true).filterNot((key, value) -> false);
    final KStream<String, Integer> stream2 = stream1.mapValues((ValueMapper<String, Integer>) Integer::valueOf);
    final KStream<String, Integer> stream3 = source2.flatMapValues((ValueMapper<String, Iterable<Integer>>) value -> Collections.singletonList(Integer.valueOf(value)));
    final KStream<String, Integer>[] streams2 = stream2.branch((key, value) -> (value % 2) == 0, (key, value) -> true);
    final KStream<String, Integer>[] streams3 = stream3.branch((key, value) -> (value % 2) == 0, (key, value) -> true);
    final int anyWindowSize = 1;
    final StreamJoined<String, Integer, Integer> joined = StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
    final KStream<String, Integer> stream4 = streams2[0].join(streams3[0], Integer::sum, JoinWindows.of(ofMillis(anyWindowSize)), joined);
    streams2[1].join(streams3[1], Integer::sum, JoinWindows.of(ofMillis(anyWindowSize)), joined);
    stream4.to("topic-5");
    streams2[1].through("topic-6").process(new MockProcessorSupplier<>());
    streams2[1].repartition().process(new MockProcessorSupplier<>());
    assertEquals(// sources
    2 + // stream1
    2 + // stream2
    1 + // stream3
    1 + 1 + // streams2
    2 + 1 + // streams3
    2 + // stream2-stream3 joins
    5 * 2 + // to
    1 + // through
    2 + // process
    1 + // repartition
    3 + // process
    1, TopologyWrapper.getInternalTopologyBuilder(builder.build()).setApplicationId("X").buildTopology().processors().size());
}