Search in sources :

Example 1 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.

the class InternalTopicIntegrationTest method shouldCompactTopicsForStateChangelogs.

@Test
public void shouldCompactTopicsForStateChangelogs() throws Exception {
    //
    // Step 1: Configure and start a simple word count topology
    //
    final Serde<String> stringSerde = Serdes.String();
    final Serde<Long> longSerde = Serdes.Long();
    final Properties streamsConfiguration = new Properties();
    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "compact-topics-integration-test");
    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    streamsConfiguration.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final KStreamBuilder builder = new KStreamBuilder();
    final KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
    final KStream<String, Long> wordCounts = textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(final String value) {
            return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
        }
    }).groupBy(MockKeyValueMapper.<String, String>SelectValueMapper()).count("Counts").toStream();
    wordCounts.to(stringSerde, longSerde, DEFAULT_OUTPUT_TOPIC);
    // Remove any state from previous test runs
    IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
    final KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    //
    // Step 2: Produce some input data to the input topic.
    //
    produceData(Arrays.asList("hello", "world", "world", "hello world"));
    //
    // Step 3: Verify the state changelog topics are compact
    //
    streams.close();
    final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "Counts"));
    assertEquals(LogConfig.Compact(), properties.getProperty(LogConfig.CleanupPolicyProp()));
}
Also used : KStreamBuilder(org.apache.kafka.streams.kstream.KStreamBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) MockKeyValueMapper(org.apache.kafka.test.MockKeyValueMapper) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Properties(java.util.Properties) Test(org.junit.Test)

Example 2 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.

the class KTableImplTest method testValueGetter.

@Test
public void testValueGetter() throws IOException {
    final KStreamBuilder builder = new KStreamBuilder();
    String topic1 = "topic1";
    String topic2 = "topic2";
    String storeName1 = "storeName1";
    String storeName2 = "storeName2";
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(stringSerde, stringSerde, topic1, storeName1);
    KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
    KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
    KTableValueGetterSupplier<String, String> getterSupplier4 = table4.valueGetterSupplier();
    driver = new KStreamTestDriver(builder, stateDir, null, null);
    // two state store should be created
    assertEquals(2, driver.allStateStores().size());
    KTableValueGetter<String, String> getter1 = getterSupplier1.get();
    getter1.init(driver.context());
    KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
    getter2.init(driver.context());
    KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
    getter3.init(driver.context());
    KTableValueGetter<String, String> getter4 = getterSupplier4.get();
    getter4.init(driver.context());
    driver.process(topic1, "A", "01");
    driver.process(topic1, "B", "01");
    driver.process(topic1, "C", "01");
    driver.flushState();
    assertEquals("01", getter1.get("A"));
    assertEquals("01", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(1), getter2.get("A"));
    assertEquals(new Integer(1), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertNull(getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("01", getter4.get("A"));
    assertEquals("01", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "02");
    driver.process(topic1, "B", "02");
    driver.flushState();
    assertEquals("02", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(2), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertEquals(new Integer(2), getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("02", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "03");
    driver.flushState();
    assertEquals("03", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(3), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("03", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", null);
    driver.flushState();
    assertNull(getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertNull(getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertNull(getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
}
Also used : KStreamBuilder(org.apache.kafka.streams.kstream.KStreamBuilder) MockKeyValueMapper(org.apache.kafka.test.MockKeyValueMapper) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Predicate(org.apache.kafka.streams.kstream.Predicate) KStreamTestDriver(org.apache.kafka.test.KStreamTestDriver) Test(org.junit.Test)

Example 3 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.

the class KTableMapValuesTest method testValueGetter.

@Test
public void testValueGetter() throws IOException {
    KStreamBuilder builder = new KStreamBuilder();
    String topic1 = "topic1";
    String topic2 = "topic2";
    String storeName1 = "storeName1";
    String storeName2 = "storeName2";
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(stringSerde, stringSerde, topic1, storeName1);
    KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
    KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
    KTableValueGetterSupplier<String, String> getterSupplier4 = table4.valueGetterSupplier();
    driver = new KStreamTestDriver(builder, stateDir, null, null);
    KTableValueGetter<String, String> getter1 = getterSupplier1.get();
    getter1.init(driver.context());
    KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
    getter2.init(driver.context());
    KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
    getter3.init(driver.context());
    KTableValueGetter<String, String> getter4 = getterSupplier4.get();
    getter4.init(driver.context());
    driver.process(topic1, "A", "01");
    driver.process(topic1, "B", "01");
    driver.process(topic1, "C", "01");
    driver.flushState();
    assertEquals("01", getter1.get("A"));
    assertEquals("01", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(1), getter2.get("A"));
    assertEquals(new Integer(1), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertNull(getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("01", getter4.get("A"));
    assertEquals("01", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "02");
    driver.process(topic1, "B", "02");
    driver.flushState();
    assertEquals("02", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(2), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertEquals(new Integer(2), getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("02", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", "03");
    driver.flushState();
    assertEquals("03", getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertEquals(new Integer(3), getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertEquals("03", getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
    driver.process(topic1, "A", null);
    driver.flushState();
    assertNull(getter1.get("A"));
    assertEquals("02", getter1.get("B"));
    assertEquals("01", getter1.get("C"));
    assertNull(getter2.get("A"));
    assertEquals(new Integer(2), getter2.get("B"));
    assertEquals(new Integer(1), getter2.get("C"));
    assertNull(getter3.get("A"));
    assertEquals(new Integer(2), getter3.get("B"));
    assertNull(getter3.get("C"));
    assertNull(getter4.get("A"));
    assertEquals("02", getter4.get("B"));
    assertEquals("01", getter4.get("C"));
}
Also used : KStreamBuilder(org.apache.kafka.streams.kstream.KStreamBuilder) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Predicate(org.apache.kafka.streams.kstream.Predicate) KStreamTestDriver(org.apache.kafka.test.KStreamTestDriver) Test(org.junit.Test)

Example 4 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class InternalTopicIntegrationTest method shouldCompactAndDeleteTopicsForWindowStoreChangelogs.

@Test
public void shouldCompactAndDeleteTopicsForWindowStoreChangelogs() throws Exception {
    final String appID = APP_ID + "-compact-delete";
    streamsProp.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
    // 
    // Step 1: Configure and start a simple word count topology
    // 
    StreamsBuilder builder = new StreamsBuilder();
    KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
    final int durationMs = 2000;
    textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(final String value) {
            return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
        }
    }).groupBy(MockMapper.<String, String>selectValueMapper()).windowedBy(TimeWindows.of(1000).until(2000)).count(Materialized.<String, Long, WindowStore<org.apache.kafka.common.utils.Bytes, byte[]>>as("CountWindows"));
    KafkaStreams streams = new KafkaStreams(builder.build(), streamsProp);
    streams.start();
    // 
    // Step 2: Produce some input data to the input topic.
    // 
    produceData(Arrays.asList("hello", "world", "world", "hello world"));
    // 
    // Step 3: Verify the state changelog topics are compact
    // 
    streams.close();
    final Properties properties = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "CountWindows"));
    final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
    assertEquals(2, policies.size());
    assertTrue(policies.contains(LogConfig.Compact()));
    assertTrue(policies.contains(LogConfig.Delete()));
    // retention should be 1 day + the window duration
    final long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
    assertEquals(retention, Long.parseLong(properties.getProperty(LogConfig.RetentionMsProp())));
    final Properties repartitionProps = getTopicProperties(appID + "-CountWindows-repartition");
    assertEquals(LogConfig.Delete(), repartitionProps.getProperty(LogConfig.CleanupPolicyProp()));
    assertEquals(4, repartitionProps.size());
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) Properties(java.util.Properties) IntegrationTest(org.apache.kafka.test.IntegrationTest) Test(org.junit.Test)

Example 5 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method createCountStream.

/**
 * Creates a typical word count topology
 */
private KafkaStreams createCountStream(final String inputTopic, final String outputTopic, final String windowOutputTopic, final String storeName, final String windowStoreName, final Properties streamsConfiguration) {
    final StreamsBuilder builder = new StreamsBuilder();
    final Serde<String> stringSerde = Serdes.String();
    final KStream<String, String> textLines = builder.stream(inputTopic, Consumed.with(stringSerde, stringSerde));
    final KGroupedStream<String, String> groupedByWord = textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(final String value) {
            return Arrays.asList(value.split("\\W+"));
        }
    }).groupBy(MockMapper.<String, String>selectValueMapper());
    // Create a State Store for the all time word count
    groupedByWord.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(storeName + "-" + inputTopic)).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    // Create a Windowed State Store that contains the word count for every 1 minute
    groupedByWord.windowedBy(TimeWindows.of(WINDOW_SIZE)).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as(windowStoreName + "-" + inputTopic)).toStream(new KeyValueMapper<Windowed<String>, Long, String>() {

        @Override
        public String apply(final Windowed<String> key, final Long value) {
            return key.key();
        }
    }).to(windowOutputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    return new KafkaStreams(builder.build(), streamsConfiguration);
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Windowed(org.apache.kafka.streams.kstream.Windowed) Bytes(org.apache.kafka.common.utils.Bytes) KafkaStreams(org.apache.kafka.streams.KafkaStreams) ValueMapper(org.apache.kafka.streams.kstream.ValueMapper) KeyValueMapper(org.apache.kafka.streams.kstream.KeyValueMapper) KeyValueMapper(org.apache.kafka.streams.kstream.KeyValueMapper)

Aggregations

ValueMapper (org.apache.kafka.streams.kstream.ValueMapper)27 Test (org.junit.Test)23 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)21 Properties (java.util.Properties)12 Predicate (org.apache.kafka.streams.kstream.Predicate)10 ArrayList (java.util.ArrayList)7 Serdes (org.apache.kafka.common.serialization.Serdes)7 Bytes (org.apache.kafka.common.utils.Bytes)7 KeyValue (org.apache.kafka.streams.KeyValue)7 Consumed (org.apache.kafka.streams.kstream.Consumed)7 Duration (java.time.Duration)6 List (java.util.List)6 KafkaStreams (org.apache.kafka.streams.KafkaStreams)6 KTable (org.apache.kafka.streams.kstream.KTable)6 Materialized (org.apache.kafka.streams.kstream.Materialized)6 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)6 MockProcessorSupplier (org.apache.kafka.test.MockProcessorSupplier)5 Set (java.util.Set)4 KeyValueTimestamp (org.apache.kafka.streams.KeyValueTimestamp)4 TestInputTopic (org.apache.kafka.streams.TestInputTopic)4