Search in sources :

Example 36 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class ProcessorTopologyTest method testPrefixScanPersistentTimestampedStoreWithCachingNoLogging.

@Test
public void testPrefixScanPersistentTimestampedStoreWithCachingNoLogging() {
    final StoreBuilder<KeyValueStore<String, String>> storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentTimestampedKeyValueStore(DEFAULT_STORE_NAME), Serdes.String(), Serdes.String()).withCachingEnabled().withLoggingDisabled();
    topology.addSource("source1", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1).addProcessor("processor1", defineWithStores(() -> new StatefulProcessor(DEFAULT_STORE_NAME), Collections.singleton(storeBuilder)), "source1").addSink("counts", OUTPUT_TOPIC_1, "processor1");
    driver = new TopologyTestDriver(topology, props);
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
    final TestOutputTopic<Integer, String> outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
    inputTopic.pipeInput("key1", "value1");
    inputTopic.pipeInput("key2", "value2");
    inputTopic.pipeInput("key3", "value3");
    inputTopic.pipeInput("key1", "value4");
    assertTrue(outputTopic1.isEmpty());
    final KeyValueStore<String, String> store = driver.getKeyValueStore(DEFAULT_STORE_NAME);
    final List<KeyValue<String, String>> results = prefixScanResults(store, DEFAULT_PREFIX);
    assertEquals("key1", results.get(0).key);
    assertEquals("value4", results.get(0).value);
    assertEquals("key2", results.get(1).key);
    assertEquals("value2", results.get(1).value);
    assertEquals("key3", results.get(2).key);
    assertEquals("value3", results.get(2).value);
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 37 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class ProcessorTopologyTest method testPrefixScanPersistentStoreWithCachingWithLoggingOldProcessor.

// testing old PAPI
@Deprecated
@Test
public void testPrefixScanPersistentStoreWithCachingWithLoggingOldProcessor() {
    final StoreBuilder<KeyValueStore<String, String>> storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(DEFAULT_STORE_NAME), Serdes.String(), Serdes.String()).withCachingEnabled().withLoggingEnabled(Collections.emptyMap());
    topology.addSource("source1", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1).addProcessor("processor1", defineWithStoresOldAPI(() -> new OldAPIStatefulProcessor(DEFAULT_STORE_NAME), Collections.singleton(storeBuilder)), "source1").addSink("counts", OUTPUT_TOPIC_1, "processor1");
    driver = new TopologyTestDriver(topology, props);
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
    final TestOutputTopic<Integer, String> outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
    inputTopic.pipeInput("key1", "value1");
    inputTopic.pipeInput("key2", "value2");
    inputTopic.pipeInput("key3", "value3");
    inputTopic.pipeInput("key1", "value4");
    assertTrue(outputTopic1.isEmpty());
    final KeyValueStore<String, String> store = driver.getKeyValueStore(DEFAULT_STORE_NAME);
    final List<KeyValue<String, String>> results = prefixScanResults(store, DEFAULT_PREFIX);
    assertEquals("key1", results.get(0).key);
    assertEquals("value4", results.get(0).value);
    assertEquals("key2", results.get(1).key);
    assertEquals("value2", results.get(1).value);
    assertEquals("key3", results.get(2).key);
    assertEquals("value3", results.get(2).value);
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 38 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class ProcessorTopologyTest method testPrefixScanLruMapWithCachingWithLoggingOldProcessor.

// testing old PAPI
@Deprecated
@Test
public void testPrefixScanLruMapWithCachingWithLoggingOldProcessor() {
    final StoreBuilder<KeyValueStore<String, String>> storeBuilder = Stores.keyValueStoreBuilder(Stores.lruMap(DEFAULT_STORE_NAME, 100), Serdes.String(), Serdes.String()).withCachingEnabled().withLoggingEnabled(Collections.emptyMap());
    topology.addSource("source1", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1).addProcessor("processor1", defineWithStoresOldAPI(() -> new OldAPIStatefulProcessor(DEFAULT_STORE_NAME), Collections.singleton(storeBuilder)), "source1").addSink("counts", OUTPUT_TOPIC_1, "processor1");
    driver = new TopologyTestDriver(topology, props);
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
    final TestOutputTopic<Integer, String> outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
    inputTopic.pipeInput("key1", "value1");
    inputTopic.pipeInput("key2", "value2");
    inputTopic.pipeInput("key3", "value3");
    inputTopic.pipeInput("key1", "value4");
    assertTrue(outputTopic1.isEmpty());
    final KeyValueStore<String, String> store = driver.getKeyValueStore(DEFAULT_STORE_NAME);
    final List<KeyValue<String, String>> results = prefixScanResults(store, DEFAULT_PREFIX);
    assertEquals("key1", results.get(0).key);
    assertEquals("value4", results.get(0).value);
    assertEquals("key2", results.get(1).key);
    assertEquals("value2", results.get(1).value);
    assertEquals("key3", results.get(2).key);
    assertEquals("value3", results.get(2).value);
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 39 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class ProcessorTopologyTest method testPrefixScanLruMapWithCachingWithLogging.

@Test
public void testPrefixScanLruMapWithCachingWithLogging() {
    final StoreBuilder<KeyValueStore<String, String>> storeBuilder = Stores.keyValueStoreBuilder(Stores.lruMap(DEFAULT_STORE_NAME, 100), Serdes.String(), Serdes.String()).withCachingEnabled().withLoggingEnabled(Collections.emptyMap());
    topology.addSource("source1", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1).addProcessor("processor1", defineWithStores(() -> new StatefulProcessor(DEFAULT_STORE_NAME), Collections.singleton(storeBuilder)), "source1").addSink("counts", OUTPUT_TOPIC_1, "processor1");
    driver = new TopologyTestDriver(topology, props);
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
    final TestOutputTopic<Integer, String> outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
    inputTopic.pipeInput("key1", "value1");
    inputTopic.pipeInput("key2", "value2");
    inputTopic.pipeInput("key3", "value3");
    inputTopic.pipeInput("key1", "value4");
    assertTrue(outputTopic1.isEmpty());
    final KeyValueStore<String, String> store = driver.getKeyValueStore(DEFAULT_STORE_NAME);
    final List<KeyValue<String, String>> results = prefixScanResults(store, DEFAULT_PREFIX);
    assertEquals("key1", results.get(0).key);
    assertEquals("value4", results.get(0).value);
    assertEquals("key2", results.get(1).key);
    assertEquals("value2", results.get(1).value);
    assertEquals("key3", results.get(2).key);
    assertEquals("value3", results.get(2).value);
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 40 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class ProcessorTopologyTest method testPrefixScanPersistentTimestampedStoreWithCachingWithLogging.

@Test
public void testPrefixScanPersistentTimestampedStoreWithCachingWithLogging() {
    final StoreBuilder<KeyValueStore<String, String>> storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentTimestampedKeyValueStore(DEFAULT_STORE_NAME), Serdes.String(), Serdes.String()).withCachingEnabled().withLoggingEnabled(Collections.emptyMap());
    topology.addSource("source1", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1).addProcessor("processor1", defineWithStores(() -> new StatefulProcessor(DEFAULT_STORE_NAME), Collections.singleton(storeBuilder)), "source1").addSink("counts", OUTPUT_TOPIC_1, "processor1");
    driver = new TopologyTestDriver(topology, props);
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
    final TestOutputTopic<Integer, String> outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
    inputTopic.pipeInput("key1", "value1");
    inputTopic.pipeInput("key2", "value2");
    inputTopic.pipeInput("key3", "value3");
    inputTopic.pipeInput("key1", "value4");
    assertTrue(outputTopic1.isEmpty());
    final KeyValueStore<String, String> store = driver.getKeyValueStore(DEFAULT_STORE_NAME);
    final List<KeyValue<String, String>> results = prefixScanResults(store, DEFAULT_PREFIX);
    assertEquals("key1", results.get(0).key);
    assertEquals("value4", results.get(0).value);
    assertEquals("key2", results.get(1).key);
    assertEquals("value2", results.get(1).value);
    assertEquals("key3", results.get(2).key);
    assertEquals("value3", results.get(2).value);
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Aggregations

KeyValueStore (org.apache.kafka.streams.state.KeyValueStore)133 Test (org.junit.Test)101 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)54 KeyValue (org.apache.kafka.streams.KeyValue)49 TopologyTestDriver (org.apache.kafka.streams.TopologyTestDriver)47 Properties (java.util.Properties)37 Bytes (org.apache.kafka.common.utils.Bytes)36 StringSerializer (org.apache.kafka.common.serialization.StringSerializer)32 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)29 KafkaStreams (org.apache.kafka.streams.KafkaStreams)28 Serdes (org.apache.kafka.common.serialization.Serdes)26 Materialized (org.apache.kafka.streams.kstream.Materialized)25 StreamsConfig (org.apache.kafka.streams.StreamsConfig)24 IntegrationTest (org.apache.kafka.test.IntegrationTest)21 KTable (org.apache.kafka.streams.kstream.KTable)20 Consumed (org.apache.kafka.streams.kstream.Consumed)19 StateStore (org.apache.kafka.streams.processor.StateStore)17 ReadOnlyKeyValueStore (org.apache.kafka.streams.state.ReadOnlyKeyValueStore)17 TestUtils (org.apache.kafka.test.TestUtils)16 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)16