Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 56 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableFilterTest method doTestSkipNullOnMaterialization.

private void doTestSkipNullOnMaterialization(final StreamsBuilder builder, final KTableImpl<String, String, String> table1, final KTableImpl<String, String, String> table2, final String topic1) {
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Topology topology = builder.build();
    topology.addProcessor("proc1", supplier, table1.name);
    topology.addProcessor("proc2", supplier, table2.name);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> stringinputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        stringinputTopic.pipeInput("A", "reject", 5L);
        stringinputTopic.pipeInput("B", "reject", 10L);
        stringinputTopic.pipeInput("C", "reject", 20L);
    }
    final List<MockApiProcessor<String, String, Void, Void>> processors = supplier.capturedProcessors(2);
    processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("reject", null), 5), new KeyValueTimestamp<>("B", new Change<>("reject", null), 10), new KeyValueTimestamp<>("C", new Change<>("reject", null), 20));
    processors.get(1).checkEmptyAndClearProcessResult();
}

Example 57 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableImplTest method shouldCreateSourceAndSinkNodesForRepartitioningTopic.

@Test
public void shouldCreateSourceAndSinkNodesForRepartitioningTopic() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String storeName1 = "storeName1";
    final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(storeName1).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()));
    table1.groupBy(MockMapper.noOpKeyValueMapper()).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.as("mock-result1"));
    table1.groupBy(MockMapper.noOpKeyValueMapper()).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.as("mock-result2"));
    final Topology topology = builder.build();
    try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(topology, props)) {
        assertEquals(3, driver.getAllStateStores().size());
        assertTopologyContainsProcessor(topology, "KSTREAM-SINK-0000000003");
        assertTopologyContainsProcessor(topology, "KSTREAM-SOURCE-0000000004");
        assertTopologyContainsProcessor(topology, "KSTREAM-SINK-0000000007");
        assertTopologyContainsProcessor(topology, "KSTREAM-SOURCE-0000000008");
        final Field valSerializerField = ((SinkNode) driver.getProcessor("KSTREAM-SINK-0000000003")).getClass().getDeclaredField("valSerializer");
        final Field valDeserializerField = ((SourceNode) driver.getProcessor("KSTREAM-SOURCE-0000000004")).getClass().getDeclaredField("valDeserializer");
        valSerializerField.setAccessible(true);
        valDeserializerField.setAccessible(true);
        assertNotNull(((ChangedSerializer) valSerializerField.get(driver.getProcessor("KSTREAM-SINK-0000000003"))).inner());
        assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.getProcessor("KSTREAM-SOURCE-0000000004"))).inner());
        assertNotNull(((ChangedSerializer) valSerializerField.get(driver.getProcessor("KSTREAM-SINK-0000000007"))).inner());
        assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.getProcessor("KSTREAM-SOURCE-0000000008"))).inner());
    }
}

Example 58 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableSourceTest method testValueGetter.

@Test
public void testValueGetter() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    @SuppressWarnings("unchecked") final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, stringConsumed, Materialized.as("store"));
    final Topology topology = builder.build();
    final KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
    final InternalTopologyBuilder topologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology);
    topologyBuilder.connectProcessorAndStateStores(table1.name, getterSupplier1.storeNames());
    try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final KTableValueGetter<String, String> getter1 = getterSupplier1.get();
        getter1.init(driver.setCurrentNodeForProcessorContext(table1.name));
        inputTopic1.pipeInput("A", "01", 10L);
        inputTopic1.pipeInput("B", "01", 20L);
        inputTopic1.pipeInput("C", "01", 15L);
        assertEquals(ValueAndTimestamp.make("01", 10L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("01", 20L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", "02", 30L);
        inputTopic1.pipeInput("B", "02", 5L);
        assertEquals(ValueAndTimestamp.make("02", 30L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", "03", 29L);
        assertEquals(ValueAndTimestamp.make("03", 29L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", null, 50L);
        inputTopic1.pipeInput("B", null, 3L);
        assertNull(getter1.get("A"));
        assertNull(getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
    }
}

Example 59 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableSourceTest method testSendingOldValue.

@Test
public void testSendingOldValue() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    @SuppressWarnings("unchecked") final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, stringConsumed);
    table1.enableSendingOldValues(true);
    assertTrue(table1.sendingOldValueEnabled());
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<String, Integer, Void, Void> proc1 = supplier.theCapturedProcessor();
        inputTopic1.pipeInput("A", "01", 10L);
        inputTopic1.pipeInput("B", "01", 20L);
        inputTopic1.pipeInput("C", "01", 15L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("01", null), 10), new KeyValueTimestamp<>("B", new Change<>("01", null), 20), new KeyValueTimestamp<>("C", new Change<>("01", null), 15));
        inputTopic1.pipeInput("A", "02", 8L);
        inputTopic1.pipeInput("B", "02", 22L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("02", "01"), 8), new KeyValueTimestamp<>("B", new Change<>("02", "01"), 22));
        inputTopic1.pipeInput("A", "03", 12L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("03", "02"), 12));
        inputTopic1.pipeInput("A", null, 15L);
        inputTopic1.pipeInput("B", null, 20L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, "03"), 15), new KeyValueTimestamp<>("B", new Change<>(null, "02"), 20));
    }
}

Example 60 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class SuppressScenarioTest method shouldSuppressIntermediateEventsWithRecordLimit.

@Test
public void shouldSuppressIntermediateEventsWithRecordLimit() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = builder.table("input", Consumed.with(STRING_SERDE, STRING_SERDE), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(STRING_SERDE, STRING_SERDE).withCachingDisabled().withLoggingDisabled()).groupBy((k, v) -> new KeyValue<>(v, k), Grouped.with(STRING_SERDE, STRING_SERDE)).count(Materialized.with(STRING_SERDE, Serdes.Long()));
    valueCounts.suppress(untilTimeLimit(ofMillis(Long.MAX_VALUE), maxRecords(1L).emitEarlyWhenFull())).toStream().to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    System.out.println(topology.describe());
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v2", 1L);
        inputTopic.pipeInput("k2", "v1", 2L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("v1", 1L, 0L), new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L), new KeyValueTimestamp<>("v1", 1L, 2L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(// consecutive updates to v1 get suppressed into only the latter.
        new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L)));
        inputTopic.pipeInput("x", "x", 3L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("x", 1L, 3L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(// now we see that last update to v1, but we won't see the update to x until it gets evicted
        new KeyValueTimestamp<>("v1", 1L, 2L)));
    }
}