Examples with ProcessorTopology org.apache.kafka.streams.processor.internals.ProcessorTopology used on opensource projects

Example 71 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class InternalStreamsBuilderTest method shouldAddGlobalTablesToEachGroup.

@Test
public void shouldAddGlobalTablesToEachGroup() {
    final String one = "globalTable";
    final String two = "globalTable2";
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materializedInternal = new MaterializedInternal<>(Materialized.as(one), builder, storePrefix);
    final GlobalKTable<String, String> globalTable = builder.globalTable("table", consumed, materializedInternal);
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materializedInternal2 = new MaterializedInternal<>(Materialized.as(two), builder, storePrefix);
    final GlobalKTable<String, String> globalTable2 = builder.globalTable("table2", consumed, materializedInternal2);
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materializedInternalNotGlobal = new MaterializedInternal<>(Materialized.as("not-global"), builder, storePrefix);
    builder.table("not-global", consumed, materializedInternalNotGlobal);
    final KeyValueMapper<String, String, String> kvMapper = (key, value) -> value;
    final KStream<String, String> stream = builder.stream(Collections.singleton("t1"), consumed);
    stream.leftJoin(globalTable, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final KStream<String, String> stream2 = builder.stream(Collections.singleton("t2"), consumed);
    stream2.leftJoin(globalTable2, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final Map<Integer, Set<String>> nodeGroups = builder.internalTopologyBuilder.nodeGroups();
    for (final Integer groupId : nodeGroups.keySet()) {
        final ProcessorTopology topology = builder.internalTopologyBuilder.buildSubtopology(groupId);
        final List<StateStore> stateStores = topology.globalStateStores();
        final Set<String> names = new HashSet<>();
        for (final StateStore stateStore : stateStores) {
            names.add(stateStore.name());
        }
        assertEquals(2, stateStores.size());
        assertTrue(names.contains(one));
        assertTrue(names.contains(two));
    }
}

Example 72 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class InternalStreamsBuilderTest method ktableShouldHaveNullTimestampExtractorWhenNoneSupplied.

@Test
public void ktableShouldHaveNullTimestampExtractorWhenNoneSupplied() {
    builder.table("topic", consumed, materialized);
    builder.buildAndOptimizeTopology();
    final ProcessorTopology processorTopology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(StreamsTestUtils.getStreamsConfig(APP_ID))).buildTopology();
    assertNull(processorTopology.source("topic").getTimestampExtractor());
}

Example 73 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class InternalStreamsBuilderTest method shouldBuildSimpleGlobalTableTopology.

@Test
public void shouldBuildSimpleGlobalTableTopology() {
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materializedInternal = new MaterializedInternal<>(Materialized.as("globalTable"), builder, storePrefix);
    builder.globalTable("table", consumed, materializedInternal);
    builder.buildAndOptimizeTopology();
    final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(StreamsTestUtils.getStreamsConfig(APP_ID))).buildGlobalStateTopology();
    final List<StateStore> stateStores = topology.globalStateStores();
    assertEquals(1, stateStores.size());
    assertEquals("globalTable", stateStores.get(0).name());
}

Example 74 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class StreamsBuilderTest method shouldNotAddThirdStateStoreIfStreamStreamJoinFixIsDisabledViaOldApi.

@SuppressWarnings("deprecation")
@Test
public void shouldNotAddThirdStateStoreIfStreamStreamJoinFixIsDisabledViaOldApi() {
    final KStream<String, String> streamOne = builder.stream(STREAM_TOPIC);
    final KStream<String, String> streamTwo = builder.stream(STREAM_TOPIC_TWO);
    streamOne.leftJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), StreamJoined.<String, String, String>as(STREAM_OPERATION_NAME).withName(STREAM_OPERATION_NAME));
    final Properties properties = new Properties();
    builder.build(properties);
    final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).buildTopology();
    assertNamesForStateStore(topology.stateStores(), STREAM_OPERATION_NAME + "-this-join-store", STREAM_OPERATION_NAME + "-outer-other-join-store");
    assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "KSTREAM-SOURCE-0000000001", STREAM_OPERATION_NAME + "-this-windowed", STREAM_OPERATION_NAME + "-other-windowed", STREAM_OPERATION_NAME + "-this-join", STREAM_OPERATION_NAME + "-outer-other-join", STREAM_OPERATION_NAME + "-merge");
}

Example 75 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class StreamsBuilderTest method shouldUseSpecifiedNameForSplitOperation.

@Test
public void shouldUseSpecifiedNameForSplitOperation() {
    builder.stream(STREAM_TOPIC).split(Named.as("branch-processor")).branch((k, v) -> true, Branched.as("-1")).branch((k, v) -> false, Branched.as("-2"));
    builder.build();
    final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).buildTopology();
    assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "branch-processor", "branch-processor-1", "branch-processor-2");
}