Examples with KeyValueStore org.apache.kafka.streams.state.KeyValueStore used on opensource projects

Example 6 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class RestoreIntegrationTest method shouldProcessDataFromStoresWithLoggingDisabled.

@Test
public void shouldProcessDataFromStoresWithLoggingDisabled() throws InterruptedException, ExecutionException {
    IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_STREAM_2, Arrays.asList(KeyValue.pair(1, 1), KeyValue.pair(2, 2), KeyValue.pair(3, 3)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), IntegerSerializer.class, IntegerSerializer.class), CLUSTER.time);
    final KeyValueBytesStoreSupplier lruMapSupplier = Stores.lruMap(INPUT_STREAM_2, 10);
    final StoreBuilder<KeyValueStore<Integer, Integer>> storeBuilder = new KeyValueStoreBuilder<>(lruMapSupplier, Serdes.Integer(), Serdes.Integer(), CLUSTER.time).withLoggingDisabled();
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.addStateStore(storeBuilder);
    final KStream<Integer, Integer> stream = streamsBuilder.stream(INPUT_STREAM_2);
    final CountDownLatch processorLatch = new CountDownLatch(3);
    stream.process(new ProcessorSupplier<Integer, Integer>() {

        @Override
        public Processor<Integer, Integer> get() {
            return new KeyValueStoreProcessor(INPUT_STREAM_2, processorLatch);
        }
    }, INPUT_STREAM_2);
    final Topology topology = streamsBuilder.build();
    kafkaStreams = new KafkaStreams(topology, props(applicationId + "-logging-disabled"));
    final CountDownLatch latch = new CountDownLatch(1);
    kafkaStreams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                latch.countDown();
            }
        }
    });
    kafkaStreams.start();
    latch.await(30, TimeUnit.SECONDS);
    assertTrue(processorLatch.await(30, TimeUnit.SECONDS));
}

Example 7 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testRepartition.

@Test
public void testRepartition() throws NoSuchFieldException, IllegalAccessException {
    String topic1 = "topic1";
    String storeName1 = "storeName1";
    final StreamsBuilder builder = new StreamsBuilder();
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(storeName1).withKeySerde(stringSerde).withValueSerde(stringSerde));
    table1.groupBy(MockMapper.<String, String>noOpKeyValueMapper()).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, "mock-result1");
    table1.groupBy(MockMapper.<String, String>noOpKeyValueMapper()).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, "mock-result2");
    driver.setUp(builder, stateDir, stringSerde, stringSerde);
    driver.setTime(0L);
    // three state store should be created, one for source, one for aggregate and one for reduce
    assertEquals(3, driver.allStateStores().size());
    // contains the corresponding repartition source / sink nodes
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SINK-0000000003"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SOURCE-0000000004"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SINK-0000000007"));
    assertTrue(driver.allProcessorNames().contains("KSTREAM-SOURCE-0000000008"));
    Field valSerializerField = ((SinkNode) driver.processor("KSTREAM-SINK-0000000003")).getClass().getDeclaredField("valSerializer");
    Field valDeserializerField = ((SourceNode) driver.processor("KSTREAM-SOURCE-0000000004")).getClass().getDeclaredField("valDeserializer");
    valSerializerField.setAccessible(true);
    valDeserializerField.setAccessible(true);
    assertNotNull(((ChangedSerializer) valSerializerField.get(driver.processor("KSTREAM-SINK-0000000003"))).inner());
    assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.processor("KSTREAM-SOURCE-0000000004"))).inner());
    assertNotNull(((ChangedSerializer) valSerializerField.get(driver.processor("KSTREAM-SINK-0000000007"))).inner());
    assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.processor("KSTREAM-SOURCE-0000000008"))).inner());
}

Example 8 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method shouldAddGlobalTablesToEachGroup.

@Test
public void shouldAddGlobalTablesToEachGroup() throws Exception {
    final String one = "globalTable";
    final String two = "globalTable2";
    final GlobalKTable<String, String> globalTable = builder.globalTable("table", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(one), builder, storePrefix));
    final GlobalKTable<String, String> globalTable2 = builder.globalTable("table2", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(two), builder, storePrefix));
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("not-global"), builder, storePrefix);
    builder.table("not-global", consumed, materialized);
    final KeyValueMapper<String, String, String> kvMapper = new KeyValueMapper<String, String, String>() {

        @Override
        public String apply(final String key, final String value) {
            return value;
        }
    };
    final KStream<String, String> stream = builder.stream(Collections.singleton("t1"), consumed);
    stream.leftJoin(globalTable, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final KStream<String, String> stream2 = builder.stream(Collections.singleton("t2"), consumed);
    stream2.leftJoin(globalTable2, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final Map<Integer, Set<String>> nodeGroups = builder.internalTopologyBuilder.nodeGroups();
    for (Integer groupId : nodeGroups.keySet()) {
        final ProcessorTopology topology = builder.internalTopologyBuilder.build(groupId);
        final List<StateStore> stateStores = topology.globalStateStores();
        final Set<String> names = new HashSet<>();
        for (StateStore stateStore : stateStores) {
            names.add(stateStore.name());
        }
        assertEquals(2, stateStores.size());
        assertTrue(names.contains(one));
        assertTrue(names.contains(two));
    }
}

Example 9 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable.

@Test
public void shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable() throws Exception {
    KTable table1 = builder.table("topic2", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(null, null), builder, storePrefix));
    final ProcessorTopology topology = builder.internalTopologyBuilder.build(null);
    assertEquals(1, topology.stateStores().size());
    final String storeName = "prefix-STATE-STORE-0000000000";
    assertEquals(storeName, topology.stateStores().get(0).name());
    assertEquals(1, topology.storeToChangelogTopic().size());
    assertEquals("topic2", topology.storeToChangelogTopic().get(storeName));
    assertNull(table1.queryableStoreName());
}

Example 10 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class InternalStreamsBuilderTest method shouldBuildSimpleGlobalTableTopology.

@Test
public void shouldBuildSimpleGlobalTableTopology() throws Exception {
    builder.globalTable("table", consumed, new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("globalTable"), builder, storePrefix));
    final ProcessorTopology topology = builder.internalTopologyBuilder.buildGlobalStateTopology();
    final List<StateStore> stateStores = topology.globalStateStores();
    assertEquals(1, stateStores.size());
    assertEquals("globalTable", stateStores.get(0).name());
}