Examples with KeyValueStore org.apache.kafka.streams.state.KeyValueStore used on opensource projects

Example 1 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class SimpleBenchmark method createKafkaStreamsWithStateStore.

private KafkaStreams createKafkaStreamsWithStateStore(String topic, final CountDownLatch latch, boolean enableCaching) {
    Properties props = setStreamProperties("simple-benchmark-streams-with-store" + enableCaching);
    KStreamBuilder builder = new KStreamBuilder();
    if (enableCaching) {
        builder.addStateStore(Stores.create("store").withIntegerKeys().withByteArrayValues().persistent().enableCaching().build());
    } else {
        builder.addStateStore(Stores.create("store").withIntegerKeys().withByteArrayValues().persistent().build());
    }
    KStream<Integer, byte[]> source = builder.stream(INTEGER_SERDE, BYTE_SERDE, topic);
    source.process(new ProcessorSupplier<Integer, byte[]>() {

        @Override
        public Processor<Integer, byte[]> get() {
            return new AbstractProcessor<Integer, byte[]>() {

                KeyValueStore<Integer, byte[]> store;

                @SuppressWarnings("unchecked")
                @Override
                public void init(ProcessorContext context) {
                    store = (KeyValueStore<Integer, byte[]>) context.getStateStore("store");
                }

                @Override
                public void process(Integer key, byte[] value) {
                    store.put(key, value);
                    processedRecords++;
                    processedBytes += value.length + Integer.SIZE;
                    if (processedRecords == numRecords) {
                        latch.countDown();
                    }
                }

                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    }, "store");
    return createKafkaStreamsWithExceptionHandler(builder, props);
}

Example 2 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project kafka by apache.

the class InMemoryKeyValueStoreTest method createKeyValueStore.

@SuppressWarnings("unchecked")
@Override
protected <K, V> KeyValueStore<K, V> createKeyValueStore(ProcessorContext context, Class<K> keyClass, Class<V> valueClass, boolean useContextSerdes) {
    StateStoreSupplier supplier;
    if (useContextSerdes) {
        supplier = Stores.create("my-store").withKeys(context.keySerde()).withValues(context.valueSerde()).inMemory().build();
    } else {
        supplier = Stores.create("my-store").withKeys(keyClass).withValues(valueClass).inMemory().build();
    }
    KeyValueStore<K, V> store = (KeyValueStore<K, V>) supplier.get();
    store.init(context, store);
    return store;
}

Example 3 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class GlobalKTableIntegrationTest method before.

@Before
public void before() throws InterruptedException {
    testNo++;
    builder = new StreamsBuilder();
    createTopics();
    streamsConfiguration = new Properties();
    final String applicationId = "globalTableTopic-table-test-" + testNo;
    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
    streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    streamsConfiguration.put(IntegrationTestUtils.INTERNAL_LEAVE_GROUP_ON_CLOSE, true);
    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
    globalTable = builder.globalTable(globalTableTopic, Consumed.with(Serdes.Long(), Serdes.String()), Materialized.<Long, String, KeyValueStore<Bytes, byte[]>>as(globalStore).withKeySerde(Serdes.Long()).withValueSerde(Serdes.String()));
    final Consumed<String, Long> stringLongConsumed = Consumed.with(Serdes.String(), Serdes.Long());
    stream = builder.stream(streamTopic, stringLongConsumed);
    foreachAction = new ForeachAction<String, String>() {

        @Override
        public void apply(final String key, final String value) {
            results.put(key, value);
        }
    };
}

Example 4 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method shouldBeAbleToQueryMapValuesState.

@Test
public void shouldBeAbleToQueryMapValuesState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, String>> batch1 = new HashSet<>(Arrays.asList(new KeyValue<>(keys[0], "1"), new KeyValue<>(keys[1], "1"), new KeyValue<>(keys[2], "3"), new KeyValue<>(keys[3], "5"), new KeyValue<>(keys[4], "2")));
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), mockTime);
    final KTable<String, String> t1 = builder.table(streamOne);
    final KTable<String, Long> t2 = t1.mapValues(new ValueMapper<String, Long>() {

        @Override
        public Long apply(final String value) {
            return Long.valueOf(value);
        }
    }, Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("queryMapValues").withValueSerde(Serdes.Long()));
    t2.toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    kafkaStreams.start();
    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
    final ReadOnlyKeyValueStore<String, Long> myMapStore = kafkaStreams.store("queryMapValues", QueryableStoreTypes.<String, Long>keyValueStore());
    for (final KeyValue<String, String> batchEntry : batch1) {
        assertEquals(myMapStore.get(batchEntry.key), Long.valueOf(batchEntry.value));
    }
}

Example 5 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class StreamsBuilderTest method shouldAllowJoinMaterializedJoinedKTable.

@Test
public void shouldAllowJoinMaterializedJoinedKTable() {
    final KTable<Bytes, String> table1 = builder.table("table-topic1");
    final KTable<Bytes, String> table2 = builder.table("table-topic2");
    builder.<Bytes, String>stream("stream-topic").join(table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Materialized.<Bytes, String, KeyValueStore<Bytes, byte[]>>as("store")), MockValueJoiner.TOSTRING_JOINER);
    driver.setUp(builder, TestUtils.tempDirectory());
    ProcessorTopology topology = builder.internalTopologyBuilder.build();
    assertThat(topology.stateStores().size(), equalTo(3));
    assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000010"), equalTo(Collections.singleton("store")));
    assertThat(topology.processorConnectedStateStores("KTABLE-MERGE-0000000007"), equalTo(Collections.singleton("store")));
}