use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.
the class InternalTopicIntegrationTest method shouldCompactTopicsForStateChangelogs.
@Test
public void shouldCompactTopicsForStateChangelogs() throws Exception {
//
// Step 1: Configure and start a simple word count topology
//
final Serde<String> stringSerde = Serdes.String();
final Serde<Long> longSerde = Serdes.Long();
final Properties streamsConfiguration = new Properties();
streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "compact-topics-integration-test");
streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
streamsConfiguration.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
streamsConfiguration.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
final KStreamBuilder builder = new KStreamBuilder();
final KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
final KStream<String, Long> wordCounts = textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {
@Override
public Iterable<String> apply(final String value) {
return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
}
}).groupBy(MockKeyValueMapper.<String, String>SelectValueMapper()).count("Counts").toStream();
wordCounts.to(stringSerde, longSerde, DEFAULT_OUTPUT_TOPIC);
// Remove any state from previous test runs
IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
final KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
streams.start();
//
// Step 2: Produce some input data to the input topic.
//
produceData(Arrays.asList("hello", "world", "world", "hello world"));
//
// Step 3: Verify the state changelog topics are compact
//
streams.close();
final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "Counts"));
assertEquals(LogConfig.Compact(), properties.getProperty(LogConfig.CleanupPolicyProp()));
}
use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.
the class KTableImplTest method testValueGetter.
@Test
public void testValueGetter() throws IOException {
final KStreamBuilder builder = new KStreamBuilder();
String topic1 = "topic1";
String topic2 = "topic2";
String storeName1 = "storeName1";
String storeName2 = "storeName2";
KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(stringSerde, stringSerde, topic1, storeName1);
KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {
@Override
public Integer apply(String value) {
return new Integer(value);
}
});
KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {
@Override
public boolean test(String key, Integer value) {
return (value % 2) == 0;
}
});
KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
KTableValueGetterSupplier<String, String> getterSupplier4 = table4.valueGetterSupplier();
driver = new KStreamTestDriver(builder, stateDir, null, null);
// two state store should be created
assertEquals(2, driver.allStateStores().size());
KTableValueGetter<String, String> getter1 = getterSupplier1.get();
getter1.init(driver.context());
KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
getter2.init(driver.context());
KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
getter3.init(driver.context());
KTableValueGetter<String, String> getter4 = getterSupplier4.get();
getter4.init(driver.context());
driver.process(topic1, "A", "01");
driver.process(topic1, "B", "01");
driver.process(topic1, "C", "01");
driver.flushState();
assertEquals("01", getter1.get("A"));
assertEquals("01", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(1), getter2.get("A"));
assertEquals(new Integer(1), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertNull(getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("01", getter4.get("A"));
assertEquals("01", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", "02");
driver.process(topic1, "B", "02");
driver.flushState();
assertEquals("02", getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(2), getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertEquals(new Integer(2), getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("02", getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", "03");
driver.flushState();
assertEquals("03", getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(3), getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("03", getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", null);
driver.flushState();
assertNull(getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertNull(getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertNull(getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
}
use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.
the class KTableMapValuesTest method testValueGetter.
@Test
public void testValueGetter() throws IOException {
KStreamBuilder builder = new KStreamBuilder();
String topic1 = "topic1";
String topic2 = "topic2";
String storeName1 = "storeName1";
String storeName2 = "storeName2";
KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(stringSerde, stringSerde, topic1, storeName1);
KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {
@Override
public Integer apply(String value) {
return new Integer(value);
}
});
KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table2.filter(new Predicate<String, Integer>() {
@Override
public boolean test(String key, Integer value) {
return (value % 2) == 0;
}
});
KTableImpl<String, String, String> table4 = (KTableImpl<String, String, String>) table1.through(stringSerde, stringSerde, topic2, storeName2);
KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
KTableValueGetterSupplier<String, String> getterSupplier4 = table4.valueGetterSupplier();
driver = new KStreamTestDriver(builder, stateDir, null, null);
KTableValueGetter<String, String> getter1 = getterSupplier1.get();
getter1.init(driver.context());
KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
getter2.init(driver.context());
KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
getter3.init(driver.context());
KTableValueGetter<String, String> getter4 = getterSupplier4.get();
getter4.init(driver.context());
driver.process(topic1, "A", "01");
driver.process(topic1, "B", "01");
driver.process(topic1, "C", "01");
driver.flushState();
assertEquals("01", getter1.get("A"));
assertEquals("01", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(1), getter2.get("A"));
assertEquals(new Integer(1), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertNull(getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("01", getter4.get("A"));
assertEquals("01", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", "02");
driver.process(topic1, "B", "02");
driver.flushState();
assertEquals("02", getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(2), getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertEquals(new Integer(2), getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("02", getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", "03");
driver.flushState();
assertEquals("03", getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertEquals(new Integer(3), getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertEquals("03", getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
driver.process(topic1, "A", null);
driver.flushState();
assertNull(getter1.get("A"));
assertEquals("02", getter1.get("B"));
assertEquals("01", getter1.get("C"));
assertNull(getter2.get("A"));
assertEquals(new Integer(2), getter2.get("B"));
assertEquals(new Integer(1), getter2.get("C"));
assertNull(getter3.get("A"));
assertEquals(new Integer(2), getter3.get("B"));
assertNull(getter3.get("C"));
assertNull(getter4.get("A"));
assertEquals("02", getter4.get("B"));
assertEquals("01", getter4.get("C"));
}
use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.
the class InternalTopicIntegrationTest method shouldCompactAndDeleteTopicsForWindowStoreChangelogs.
@Test
public void shouldCompactAndDeleteTopicsForWindowStoreChangelogs() throws Exception {
final String appID = APP_ID + "-compact-delete";
streamsProp.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
//
// Step 1: Configure and start a simple word count topology
//
StreamsBuilder builder = new StreamsBuilder();
KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
final int durationMs = 2000;
textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {
@Override
public Iterable<String> apply(final String value) {
return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
}
}).groupBy(MockMapper.<String, String>selectValueMapper()).windowedBy(TimeWindows.of(1000).until(2000)).count(Materialized.<String, Long, WindowStore<org.apache.kafka.common.utils.Bytes, byte[]>>as("CountWindows"));
KafkaStreams streams = new KafkaStreams(builder.build(), streamsProp);
streams.start();
//
// Step 2: Produce some input data to the input topic.
//
produceData(Arrays.asList("hello", "world", "world", "hello world"));
//
// Step 3: Verify the state changelog topics are compact
//
streams.close();
final Properties properties = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "CountWindows"));
final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
assertEquals(2, policies.size());
assertTrue(policies.contains(LogConfig.Compact()));
assertTrue(policies.contains(LogConfig.Delete()));
// retention should be 1 day + the window duration
final long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
assertEquals(retention, Long.parseLong(properties.getProperty(LogConfig.RetentionMsProp())));
final Properties repartitionProps = getTopicProperties(appID + "-CountWindows-repartition");
assertEquals(LogConfig.Delete(), repartitionProps.getProperty(LogConfig.CleanupPolicyProp()));
assertEquals(4, repartitionProps.size());
}
use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.
the class QueryableStateIntegrationTest method createCountStream.
/**
* Creates a typical word count topology
*/
private KafkaStreams createCountStream(final String inputTopic, final String outputTopic, final String windowOutputTopic, final String storeName, final String windowStoreName, final Properties streamsConfiguration) {
final StreamsBuilder builder = new StreamsBuilder();
final Serde<String> stringSerde = Serdes.String();
final KStream<String, String> textLines = builder.stream(inputTopic, Consumed.with(stringSerde, stringSerde));
final KGroupedStream<String, String> groupedByWord = textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {
@Override
public Iterable<String> apply(final String value) {
return Arrays.asList(value.split("\\W+"));
}
}).groupBy(MockMapper.<String, String>selectValueMapper());
// Create a State Store for the all time word count
groupedByWord.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(storeName + "-" + inputTopic)).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
// Create a Windowed State Store that contains the word count for every 1 minute
groupedByWord.windowedBy(TimeWindows.of(WINDOW_SIZE)).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as(windowStoreName + "-" + inputTopic)).toStream(new KeyValueMapper<Windowed<String>, Long, String>() {
@Override
public String apply(final Windowed<String> key, final Long value) {
return key.key();
}
}).to(windowOutputTopic, Produced.with(Serdes.String(), Serdes.Long()));
return new KafkaStreams(builder.build(), streamsConfiguration);
}
Aggregations