Examples with KeyValueStore org.apache.kafka.streams.state.KeyValueStore used on opensource projects

Example 76 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class StateStoreTestUtils method newKeyValueStore.

public static <K, V> KeyValueStore<K, V> newKeyValueStore(final String name, final String applicationId, final Class<K> keyType, final Class<V> valueType) {
    final InMemoryKeyValueStoreSupplier<K, V> supplier = new InMemoryKeyValueStoreSupplier<>(name, null, null, new MockTime(), false, Collections.<String, String>emptyMap());
    final StateStore stateStore = supplier.get();
    stateStore.init(new InternalMockProcessorContext(StateSerdes.withBuiltinTypes(ProcessorStateManager.storeChangelogTopic(applicationId, name), keyType, valueType), new NoOpRecordCollector()), stateStore);
    return (KeyValueStore<K, V>) stateStore;
}

Example 77 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class MockProcessorContextTest method shouldStoreAndReturnStateStores.

@Test
public void shouldStoreAndReturnStateStores() {
    final AbstractProcessor<String, Long> processor = new AbstractProcessor<String, Long>() {

        @Override
        public void process(final String key, final Long value) {
            // noinspection unchecked
            final KeyValueStore<String, Long> stateStore = (KeyValueStore<String, Long>) context().getStateStore("my-state");
            stateStore.put(key, (stateStore.get(key) == null ? 0 : stateStore.get(key)) + value);
            stateStore.put("all", (stateStore.get("all") == null ? 0 : stateStore.get("all")) + value);
        }
    };
    final MockProcessorContext context = new MockProcessorContext();
    final KeyValueStore<String, Long> store = new InMemoryKeyValueStore<>("my-state", Serdes.String(), Serdes.Long());
    context.register(store, false, null);
    store.init(context, store);
    processor.init(context);
    processor.process("foo", 5L);
    processor.process("bar", 50L);
    assertEquals(5L, (long) store.get("foo"));
    assertEquals(50L, (long) store.get("bar"));
    assertEquals(55L, (long) store.get("all"));
}

Example 78 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class InMemoryKeyValueLoggedStoreTest method createKeyValueStore.

@SuppressWarnings("unchecked")
@Override
protected <K, V> KeyValueStore<K, V> createKeyValueStore(final ProcessorContext context) {
    final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), (Serde<K>) context.keySerde(), (Serde<V>) context.valueSerde()).withLoggingEnabled(Collections.singletonMap("retention.ms", "1000"));
    final StateStore store = storeBuilder.build();
    store.init(context, store);
    return (KeyValueStore<K, V>) store;
}

Example 79 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class TopologyTestDriverTest method shouldCleanUpPersistentStateStoresOnClose.

@Test
public void shouldCleanUpPersistentStateStoresOnClose() {
    final Topology topology = new Topology();
    topology.addSource("sourceProcessor", "input-topic");
    topology.addProcessor("storeProcessor", new ProcessorSupplier() {

        @Override
        public Processor get() {
            return new Processor<String, Long>() {

                private KeyValueStore<String, Long> store;

                @Override
                public void init(final ProcessorContext context) {
                    // noinspection unchecked
                    this.store = (KeyValueStore<String, Long>) context.getStateStore("storeProcessorStore");
                }

                @Override
                public void process(final String key, final Long value) {
                    store.put(key, value);
                }

                @Override
                public void punctuate(final long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    }, "sourceProcessor");
    topology.addStateStore(Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("storeProcessorStore"), Serdes.String(), Serdes.Long()), "storeProcessor");
    final Properties config = new Properties();
    config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-TopologyTestDriver-cleanup");
    config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234");
    config.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());
    {
        final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config);
        Assert.assertNull(testDriver.getKeyValueStore("storeProcessorStore").get("a"));
        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L));
        Assert.assertEquals(1L, testDriver.getKeyValueStore("storeProcessorStore").get("a"));
        testDriver.close();
    }
    {
        final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config);
        Assert.assertNull("Closing the prior test driver should have cleaned up this store and value.", testDriver.getKeyValueStore("storeProcessorStore").get("a"));
    }
}

Example 80 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class EosIntegrationTest method getKafkaStreams.

private KafkaStreams getKafkaStreams(final boolean withState, final String appDir, final int numberOfStreamsThreads) {
    commitRequested = new AtomicInteger(0);
    errorInjected = new AtomicBoolean(false);
    gcInjected = new AtomicBoolean(false);
    final StreamsBuilder builder = new StreamsBuilder();
    String[] storeNames = null;
    if (withState) {
        storeNames = new String[] { storeName };
        final StoreBuilder<KeyValueStore<Long, Long>> storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(storeName), Serdes.Long(), Serdes.Long()).withCachingEnabled();
        builder.addStateStore(storeBuilder);
    }
    final KStream<Long, Long> input = builder.stream(MULTI_PARTITION_INPUT_TOPIC);
    input.transform(new TransformerSupplier<Long, Long, KeyValue<Long, Long>>() {

        @SuppressWarnings("unchecked")
        @Override
        public Transformer<Long, Long, KeyValue<Long, Long>> get() {
            return new Transformer<Long, Long, KeyValue<Long, Long>>() {

                ProcessorContext context;

                KeyValueStore<Long, Long> state = null;

                @Override
                public void init(final ProcessorContext context) {
                    this.context = context;
                    if (withState) {
                        state = (KeyValueStore<Long, Long>) context.getStateStore(storeName);
                    }
                }

                @Override
                public KeyValue<Long, Long> transform(final Long key, final Long value) {
                    if (errorInjected.compareAndSet(true, false)) {
                        // only tries to fail once on one of the task
                        throw new RuntimeException("Injected test exception.");
                    }
                    if (gcInjected.compareAndSet(true, false)) {
                        while (doGC) {
                            try {
                                Thread.sleep(100);
                            } catch (final InterruptedException e) {
                                throw new RuntimeException(e);
                            }
                        }
                    }
                    if ((value + 1) % 10 == 0) {
                        context.commit();
                        commitRequested.incrementAndGet();
                    }
                    if (state != null) {
                        Long sum = state.get(key);
                        if (sum == null) {
                            sum = value;
                        } else {
                            sum += value;
                        }
                        state.put(key, sum);
                        context.forward(key, sum);
                        return null;
                    }
                    return new KeyValue<>(key, value);
                }

                @Override
                public KeyValue<Long, Long> punctuate(final long timestamp) {
                    return null;
                }

                @Override
                public void close() {
                }
            };
        }
    }, storeNames).to(SINGLE_PARTITION_OUTPUT_TOPIC);
    final KafkaStreams streams = new KafkaStreams(builder.build(), StreamsTestUtils.getStreamsConfig(applicationId, CLUSTER.bootstrapServers(), Serdes.LongSerde.class.getName(), Serdes.LongSerde.class.getName(), new Properties() {

        {
            put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
            put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, numberOfStreamsThreads);
            put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, -1);
            put(StreamsConfig.consumerPrefix(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), 5 * 1000);
            put(StreamsConfig.consumerPrefix(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), 5 * 1000 - 1);
            put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), MAX_POLL_INTERVAL_MS);
            put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
            put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath() + File.separator + appDir);
            put(StreamsConfig.APPLICATION_SERVER_CONFIG, "dummy:2142");
        }
    }));
    streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

        @Override
        public void uncaughtException(final Thread t, final Throwable e) {
            if (uncaughtException != null) {
                e.printStackTrace(System.err);
                fail("Should only get one uncaught exception from Streams.");
            }
            uncaughtException = e;
        }
    });
    return streams;
}