Examples with ProcessorTopology org.apache.kafka.streams.processor.internals.ProcessorTopology used on opensource projects

Example 11 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project apache-kafka-on-k8s by banzaicloud.

the class KStreamImplTest method shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningTopicCreated.

@Test
public // TODO: this test should be refactored when we removed KStreamBuilder so that the created Topology contains internal topics as well
void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningTopicCreated() {
    final KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> kStream = builder.stream(stringSerde, stringSerde, "topic-1");
    ValueJoiner<String, String, String> valueJoiner = MockValueJoiner.instance(":");
    long windowSize = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS);
    final KStream<String, String> stream = kStream.map(new KeyValueMapper<String, String, KeyValue<? extends String, ? extends String>>() {

        @Override
        public KeyValue<? extends String, ? extends String> apply(String key, String value) {
            return KeyValue.pair(value, value);
        }
    });
    stream.join(kStream, valueJoiner, JoinWindows.of(windowSize).until(3 * windowSize), Joined.with(Serdes.String(), Serdes.String(), Serdes.String())).to(Serdes.String(), Serdes.String(), "output-topic");
    ProcessorTopology processorTopology = builder.setApplicationId("X").build(null);
    SourceNode originalSourceNode = processorTopology.source("topic-1");
    for (SourceNode sourceNode : processorTopology.sources()) {
        if (sourceNode.name().equals(originalSourceNode.name())) {
            assertEquals(sourceNode.getTimestampExtractor(), null);
        } else {
            assertThat(sourceNode.getTimestampExtractor(), instanceOf(FailOnInvalidTimestamp.class));
        }
    }
}

Example 12 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method shouldAddGlobalTablesToEachGroup.

@Test
public void shouldAddGlobalTablesToEachGroup() throws Exception {
    final String one = "globalTable";
    final String two = "globalTable2";
    final GlobalKTable<String, String> globalTable = builder.globalTable("table", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(one), builder, storePrefix));
    final GlobalKTable<String, String> globalTable2 = builder.globalTable("table2", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(two), builder, storePrefix));
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("not-global"), builder, storePrefix);
    builder.table("not-global", consumed, materialized);
    final KeyValueMapper<String, String, String> kvMapper = new KeyValueMapper<String, String, String>() {

        @Override
        public String apply(final String key, final String value) {
            return value;
        }
    };
    final KStream<String, String> stream = builder.stream(Collections.singleton("t1"), consumed);
    stream.leftJoin(globalTable, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final KStream<String, String> stream2 = builder.stream(Collections.singleton("t2"), consumed);
    stream2.leftJoin(globalTable2, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final Map<Integer, Set<String>> nodeGroups = builder.internalTopologyBuilder.nodeGroups();
    for (Integer groupId : nodeGroups.keySet()) {
        final ProcessorTopology topology = builder.internalTopologyBuilder.build(groupId);
        final List<StateStore> stateStores = topology.globalStateStores();
        final Set<String> names = new HashSet<>();
        for (StateStore stateStore : stateStores) {
            names.add(stateStore.name());
        }
        assertEquals(2, stateStores.size());
        assertTrue(names.contains(one));
        assertTrue(names.contains(two));
    }
}

Example 13 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable.

@Test
public void shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable() throws Exception {
    KTable table1 = builder.table("topic2", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(null, null), builder, storePrefix));
    final ProcessorTopology topology = builder.internalTopologyBuilder.build(null);
    assertEquals(1, topology.stateStores().size());
    final String storeName = "prefix-STATE-STORE-0000000000";
    assertEquals(storeName, topology.stateStores().get(0).name());
    assertEquals(1, topology.storeToChangelogTopic().size());
    assertEquals("topic2", topology.storeToChangelogTopic().get(storeName));
    assertNull(table1.queryableStoreName());
}

Example 14 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method doBuildGlobalTopologyWithAllGlobalTables.

private void doBuildGlobalTopologyWithAllGlobalTables() throws Exception {
    final ProcessorTopology topology = builder.internalTopologyBuilder.buildGlobalStateTopology();
    final List<StateStore> stateStores = topology.globalStateStores();
    final Set<String> sourceTopics = topology.sourceTopics();
    assertEquals(Utils.mkSet("table", "table2"), sourceTopics);
    assertEquals(2, stateStores.size());
}

Example 15 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method ktableShouldHaveNullTimestampExtractorWhenNoneSupplied.

@Test
public void ktableShouldHaveNullTimestampExtractorWhenNoneSupplied() throws Exception {
    builder.table("topic", consumed, materialized);
    final ProcessorTopology processorTopology = builder.internalTopologyBuilder.build(null);
    assertNull(processorTopology.source("topic").getTimestampExtractor());
}