Search in sources :

Example 6 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testValueGetter.

@Test
public void testValueGetter() {
    final StreamsBuilder builder = new StreamsBuilder();
    String topic1 = "topic1";
    String topic2 = "topic2";
    String storeName2 = "storeName2";
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
    KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
    KTableValueGetterSupplier<String, String> getterSupplier4 = table4.valueGetterSupplier();
    driver.setUp(builder, stateDir, null, null);
    // two state store should be created
    assertEquals(2, driver.allStateStores().size());
    KTableValueGetter<String, String> getter1 = getterSupplier1.get();
    getter1.init(driver.context());
    KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
    getter2.init(driver.context());
    KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
    getter3.init(driver.context());
    KTableValueGetter<String, String> getter4 = getterSupplier4.get();
    getter4.init(driver.context());
    driver.process(topic1, "A", "01");
    driver.process(topic1, "B", "01");
    driver.process(topic1, "C", "01");
    driver.flushState();
    assertEquals("01", getter1.get("A"));
    assertEquals("01", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(1), getter2.get("A"));
    assertEquals(new Integer(1), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertNull(getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("01", getter4.get("A"));
    assertEquals("01", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "02");
    driver.process(topic1, "B", "02");
    driver.flushState();
    assertEquals("02", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(2), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertEquals(new Integer(2), getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("02", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "03");
    driver.flushState();
    assertEquals("03", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(3), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("03", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", null);
    driver.flushState();
    assertNull(getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertNull(getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertNull(getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
}
Also used : ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Predicate(org.apache.kafka.streams.kstream.Predicate) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Test(org.junit.Test)

Example 7 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testStateStoreLazyEval.

@Test
public void testStateStoreLazyEval() {
    String topic1 = "topic1";
    String topic2 = "topic2";
    final StreamsBuilder builder = new StreamsBuilder();
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    builder.table(topic2, consumed);
    KTableImpl<String, String, Integer> table1Mapped = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    table1Mapped.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    driver.setUp(builder, stateDir, null, null);
    driver.setTime(0L);
    // two state stores should be created
    assertEquals(2, driver.allStateStores().size());
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Test(org.junit.Test)

Example 8 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KTableMapValuesTest method testQueryableValueGetter.

@Test
public void testQueryableValueGetter() {
    StreamsBuilder builder = new StreamsBuilder();
    String topic1 = "topic1";
    String topic2 = "topic2";
    String storeName2 = "storeName2";
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    }, Materialized.<String, Integer, KeyValueStore<Bytes, byte[]>>as("anyMapName").withValueSerde(Serdes.Integer()));
    KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    }, Materialized.<String, Integer, KeyValueStore<Bytes, byte[]>>as("anyFilterName").withValueSerde(Serdes.Integer()));
    KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
    doTestValueGetter(builder, topic1, table1, table2, table3, table4);
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Bytes(org.apache.kafka.common.utils.Bytes) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Predicate(org.apache.kafka.streams.kstream.Predicate) Test(org.junit.Test)

Example 9 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KTableMapValuesTest method testSendingOldValue.

@Test
public void testSendingOldValue() {
    StreamsBuilder builder = new StreamsBuilder();
    String topic1 = "topic1";
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    table2.enableSendingOldValues();
    MockProcessorSupplier<String, Integer> proc = new MockProcessorSupplier<>();
    builder.build().addProcessor("proc", proc, table2.name);
    driver.setUp(builder, stateDir);
    assertTrue(table1.sendingOldValueEnabled());
    assertTrue(table2.sendingOldValueEnabled());
    driver.process(topic1, "A", "01");
    driver.process(topic1, "B", "01");
    driver.process(topic1, "C", "01");
    driver.flushState();
    proc.checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)");
    driver.process(topic1, "A", "02");
    driver.process(topic1, "B", "02");
    driver.flushState();
    proc.checkAndClearProcessResult("A:(2<-1)", "B:(2<-1)");
    driver.process(topic1, "A", "03");
    driver.flushState();
    proc.checkAndClearProcessResult("A:(3<-2)");
    driver.process(topic1, "A", null);
    driver.flushState();
    proc.checkAndClearProcessResult("A:(null<-3)");
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) MockProcessorSupplier(org.apache.kafka.test.MockProcessorSupplier) Test(org.junit.Test)

Example 10 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KStreamFlatMapValuesTest method testFlatMapValues.

@Test
public void testFlatMapValues() {
    StreamsBuilder builder = new StreamsBuilder();
    ValueMapper<Number, Iterable<String>> mapper = new ValueMapper<Number, Iterable<String>>() {

        @Override
        public Iterable<String> apply(Number value) {
            ArrayList<String> result = new ArrayList<String>();
            result.add("v" + value);
            result.add("V" + value);
            return result;
        }
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, Integer> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    final MockProcessorSupplier<Integer, String> processor = new MockProcessorSupplier<>();
    stream.flatMapValues(mapper).process(processor);
    driver.setUp(builder);
    for (final int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, expectedKey);
    }
    String[] expected = { "0:v0", "0:V0", "1:v1", "1:V1", "2:v2", "2:V2", "3:v3", "3:V3" };
    assertArrayEquals(expected, processor.processed.toArray());
}
Also used : ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) ArrayList(java.util.ArrayList) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) MockProcessorSupplier(org.apache.kafka.test.MockProcessorSupplier) Test(org.junit.Test)

Aggregations

ValueMapper (org.apache.kafka.streams.kstream.ValueMapper)27 Test (org.junit.Test)23 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)21 Properties (java.util.Properties)12 Predicate (org.apache.kafka.streams.kstream.Predicate)10 ArrayList (java.util.ArrayList)7 Serdes (org.apache.kafka.common.serialization.Serdes)7 Bytes (org.apache.kafka.common.utils.Bytes)7 KeyValue (org.apache.kafka.streams.KeyValue)7 Consumed (org.apache.kafka.streams.kstream.Consumed)7 Duration (java.time.Duration)6 List (java.util.List)6 KafkaStreams (org.apache.kafka.streams.KafkaStreams)6 KTable (org.apache.kafka.streams.kstream.KTable)6 Materialized (org.apache.kafka.streams.kstream.Materialized)6 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)6 MockProcessorSupplier (org.apache.kafka.test.MockProcessorSupplier)5 Set (java.util.Set)4 KeyValueTimestamp (org.apache.kafka.streams.KeyValueTimestamp)4 TestInputTopic (org.apache.kafka.streams.TestInputTopic)4