Examples with StreamsBuilder org.apache.kafka.streams.StreamsBuilder used on opensource projects

Example 31 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testRepartition.

@Test
public void testRepartition() throws NoSuchFieldException, IllegalAccessException {
    String topic1 = "topic1";
    String storeName1 = "storeName1";
    final StreamsBuilder builder = new StreamsBuilder();
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(storeName1).withKeySerde(stringSerde).withValueSerde(stringSerde));
    table1.groupBy(MockMapper.<String, String>noOpKeyValueMapper()).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, "mock-result1");
    table1.groupBy(MockMapper.<String, String>noOpKeyValueMapper()).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, "mock-result2");
    driver.setUp(builder, stateDir, stringSerde, stringSerde);
    driver.setTime(0L);
    // three state store should be created, one for source, one for aggregate and one for reduce
    assertEquals(3, driver.allStateStores().size());
    // contains the corresponding repartition source / sink nodes
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SINK-0000000003"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SOURCE-0000000004"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SINK-0000000007"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SOURCE-0000000008"));
    Field valSerializerField = ((SinkNode) driver.processor("KSTREAM-SINK-0000000003")).getClass().getDeclaredField("valSerializer");
    Field valDeserializerField = ((SourceNode) driver.processor("KSTREAM-SOURCE-0000000004")).getClass().getDeclaredField("valDeserializer");
    valSerializerField.setAccessible(true);
    valDeserializerField.setAccessible(true);
    assertNotNull(((ChangedSerializer) valSerializerField.get(driver.processor("KSTREAM-SINK-0000000003"))).inner());
    assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.processor("KSTREAM-SOURCE-0000000004"))).inner());
    assertNotNull(((ChangedSerializer) valSerializerField.get(driver.processor("KSTREAM-SINK-0000000007"))).inner());
    assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.processor("KSTREAM-SOURCE-0000000008"))).inner());
}

Example 32 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testStateStoreLazyEval.

@Test
public void testStateStoreLazyEval() {
    String topic1 = "topic1";
    String topic2 = "topic2";
    final StreamsBuilder builder = new StreamsBuilder();
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    builder.table(topic2, consumed);
    KTableImpl<String, String, Integer> table1Mapped = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    table1Mapped.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    driver.setUp(builder, stateDir, null, null);
    driver.setTime(0L);
    // two state stores should be created
    assertEquals(2, driver.allStateStores().size());
}

Example 33 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testKTable.

@Test
public void testKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    String topic1 = "topic1";
    String topic2 = "topic2";
    String storeName2 = "storeName2";
    KTable<String, String> table1 = builder.table(topic1, consumed);
    MockProcessorSupplier<String, String> proc1 = new MockProcessorSupplier<>();
    table1.toStream().process(proc1);
    KTable<String, Integer> table2 = table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    MockProcessorSupplier<String, Integer> proc2 = new MockProcessorSupplier<>();
    table2.toStream().process(proc2);
    KTable<String, Integer> table3 = table2.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    MockProcessorSupplier<String, Integer> proc3 = new MockProcessorSupplier<>();
    table3.toStream().process(proc3);
    KTable<String, String> table4 = table1.through(stringSerde, stringSerde, topic2, storeName2);
    MockProcessorSupplier<String, String> proc4 = new MockProcessorSupplier<>();
    table4.toStream().process(proc4);
    driver.setUp(builder, stateDir);
    driver.process(topic1, "A", "01");
    driver.flushState();
    driver.process(topic1, "B", "02");
    driver.flushState();
    driver.process(topic1, "C", "03");
    driver.flushState();
    driver.process(topic1, "D", "04");
    driver.flushState();
    driver.flushState();
    assertEquals(Utils.mkList("A:01", "B:02", "C:03", "D:04"), proc1.processed);
    assertEquals(Utils.mkList("A:1", "B:2", "C:3", "D:4"), proc2.processed);
    assertEquals(Utils.mkList("A:null", "B:2", "C:null", "D:4"), proc3.processed);
    assertEquals(Utils.mkList("A:01", "B:02", "C:03", "D:04"), proc4.processed);
}

Example 34 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableKTableInnerJoinTest method testNotSendingOldValues.

@Test
public void testNotSendingOldValues() {
    final StreamsBuilder builder = new StreamsBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    final KTable<Integer, String> table1;
    final KTable<Integer, String> table2;
    final KTable<Integer, String> joined;
    final MockProcessorSupplier<Integer, String> proc;
    table1 = builder.table(topic1, consumed);
    table2 = builder.table(topic2, consumed);
    joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER);
    proc = new MockProcessorSupplier<>();
    builder.build().addProcessor("proc", proc, ((KTableImpl<?, ?, ?>) joined).name);
    doTestSendingOldValues(builder, expectedKeys, table1, table2, proc, joined, false);
}

Example 35 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableKTableInnerJoinTest method testQueryableJoin.

@Test
public void testQueryableJoin() {
    final StreamsBuilder builder = new StreamsBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    final KTable<Integer, String> table1;
    final KTable<Integer, String> table2;
    final KTable<Integer, String> joined;
    final MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    table1 = builder.table(topic1, consumed);
    table2 = builder.table(topic2, consumed);
    joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Serdes.String(), "anyQueryableName");
    joined.toStream().process(processor);
    doTestJoin(builder, expectedKeys, processor, joined);
}