Examples with KeyValueStore org.apache.kafka.streams.state.KeyValueStore used on opensource projects

Example 81 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method shouldBeAbleToQueryMapValuesAfterFilterState.

@Test
public void shouldBeAbleToQueryMapValuesAfterFilterState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, String>> batch1 = new HashSet<>(Arrays.asList(new KeyValue<>(keys[0], "1"), new KeyValue<>(keys[1], "1"), new KeyValue<>(keys[2], "3"), new KeyValue<>(keys[3], "5"), new KeyValue<>(keys[4], "2")));
    final Set<KeyValue<String, Long>> expectedBatch1 = new HashSet<>(Collections.singleton(new KeyValue<>(keys[4], 2L)));
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), mockTime);
    final Predicate<String, String> filterPredicate = new Predicate<String, String>() {

        @Override
        public boolean test(final String key, final String value) {
            return key.contains("kafka");
        }
    };
    final KTable<String, String> t1 = builder.table(streamOne);
    final KTable<String, String> t2 = t1.filter(filterPredicate, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("queryFilter"));
    final KTable<String, Long> t3 = t2.mapValues(new ValueMapper<String, Long>() {

        @Override
        public Long apply(final String value) {
            return Long.valueOf(value);
        }
    }, Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("queryMapValues").withValueSerde(Serdes.Long()));
    t3.toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    kafkaStreams.start();
    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
    final ReadOnlyKeyValueStore<String, Long> myMapStore = kafkaStreams.store("queryMapValues", QueryableStoreTypes.<String, Long>keyValueStore());
    for (final KeyValue<String, Long> expectedEntry : expectedBatch1) {
        assertEquals(myMapStore.get(expectedEntry.key), expectedEntry.value);
    }
    for (final KeyValue<String, String> batchEntry : batch1) {
        final KeyValue<String, Long> batchEntryMapValue = new KeyValue<>(batchEntry.key, Long.valueOf(batchEntry.value));
        if (!expectedBatch1.contains(batchEntryMapValue)) {
            assertNull(myMapStore.get(batchEntry.key));
        }
    }
}

Example 82 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class RestoreIntegrationTest method shouldSuccessfullyStartWhenLoggingDisabled.

@Test
public void shouldSuccessfullyStartWhenLoggingDisabled() throws InterruptedException {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, Integer> stream = builder.stream(INPUT_STREAM);
    stream.groupByKey().reduce(new Reducer<Integer>() {

        @Override
        public Integer apply(final Integer value1, final Integer value2) {
            return value1 + value2;
        }
    }, Materialized.<Integer, Integer, KeyValueStore<Bytes, byte[]>>as("reduce-store").withLoggingDisabled());
    final CountDownLatch startupLatch = new CountDownLatch(1);
    kafkaStreams = new KafkaStreams(builder.build(), props(applicationId));
    kafkaStreams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                startupLatch.countDown();
            }
        }
    });
    kafkaStreams.start();
    assertTrue(startupLatch.await(30, TimeUnit.SECONDS));
}

Example 83 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class KeyValueStoreMaterializer method materialize.

/**
 * @return  StoreBuilder
 */
public StoreBuilder<KeyValueStore<K, V>> materialize() {
    KeyValueBytesStoreSupplier supplier = (KeyValueBytesStoreSupplier) materialized.storeSupplier();
    if (supplier == null) {
        final String name = materialized.storeName();
        supplier = Stores.persistentKeyValueStore(name);
    }
    final StoreBuilder<KeyValueStore<K, V>> builder = Stores.keyValueStoreBuilder(supplier, materialized.keySerde(), materialized.valueSerde());
    if (materialized.loggingEnabled()) {
        builder.withLoggingEnabled(materialized.logConfig());
    } else {
        builder.withLoggingDisabled();
    }
    if (materialized.cachingEnabled()) {
        builder.withCachingEnabled();
    }
    return builder;
}

Example 84 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadTest method shouldUpdateStandbyTask.

@SuppressWarnings("unchecked")
@Test
public void shouldUpdateStandbyTask() {
    final String storeName1 = "count-one";
    final String storeName2 = "table-two";
    final String changelogName = applicationId + "-" + storeName1 + "-changelog";
    final TopicPartition partition1 = new TopicPartition(changelogName, 1);
    final TopicPartition partition2 = t2p1;
    internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).groupByKey().count(Materialized.<Object, Long, KeyValueStore<Bytes, byte[]>>as(storeName1));
    internalStreamsBuilder.table(topic2, new ConsumedInternal(), new MaterializedInternal(Materialized.as(storeName2), internalStreamsBuilder, ""));
    final StreamThread thread = createStreamThread(clientId, config, false);
    final MockConsumer<byte[], byte[]> restoreConsumer = clientSupplier.restoreConsumer;
    restoreConsumer.updatePartitions(changelogName, Collections.singletonList(new PartitionInfo(changelogName, 1, null, new Node[0], new Node[0])));
    restoreConsumer.assign(Utils.mkSet(partition1, partition2));
    restoreConsumer.updateEndOffsets(Collections.singletonMap(partition1, 10L));
    restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition1, 0L));
    restoreConsumer.updateEndOffsets(Collections.singletonMap(partition2, 10L));
    restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition2, 0L));
    // let the store1 be restored from 0 to 10; store2 be restored from 0 to (committed offset) 5
    clientSupplier.consumer.assign(Utils.mkSet(partition2));
    clientSupplier.consumer.commitSync(Collections.singletonMap(partition2, new OffsetAndMetadata(5L, "")));
    for (long i = 0L; i < 10L; i++) {
        restoreConsumer.addRecord(new ConsumerRecord<>(changelogName, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes()));
        restoreConsumer.addRecord(new ConsumerRecord<>(topic2, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes()));
    }
    thread.setState(StreamThread.State.RUNNING);
    thread.rebalanceListener.onPartitionsRevoked(null);
    final Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    // assign single partition
    standbyTasks.put(task1, Collections.singleton(t1p1));
    standbyTasks.put(task3, Collections.singleton(t2p1));
    thread.taskManager().setAssignmentMetadata(Collections.<TaskId, Set<TopicPartition>>emptyMap(), standbyTasks);
    thread.rebalanceListener.onPartitionsAssigned(Collections.<TopicPartition>emptyList());
    thread.runOnce(-1);
    final StandbyTask standbyTask1 = thread.taskManager().standbyTask(partition1);
    final StandbyTask standbyTask2 = thread.taskManager().standbyTask(partition2);
    final KeyValueStore<Object, Long> store1 = (KeyValueStore<Object, Long>) standbyTask1.getStore(storeName1);
    final KeyValueStore<Object, Long> store2 = (KeyValueStore<Object, Long>) standbyTask2.getStore(storeName2);
    assertEquals(10L, store1.approximateNumEntries());
    assertEquals(5L, store2.approximateNumEntries());
    assertEquals(Collections.singleton(partition2), restoreConsumer.paused());
    assertEquals(1, thread.standbyRecords().size());
    assertEquals(5, thread.standbyRecords().get(partition2).size());
}

Example 85 with KeyValueStore

use of org.apache.kafka.streams.state.KeyValueStore in project apache-kafka-on-k8s by banzaicloud.

the class GlobalStreamThreadTest method before.

@SuppressWarnings("unchecked")
@Before
public void before() {
    final MaterializedInternal<Object, Object, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<Object, Object, KeyValueStore<Bytes, byte[]>>with(null, null), new InternalNameProvider() {

        @Override
        public String newProcessorName(String prefix) {
            return "processorName";
        }

        @Override
        public String newStoreName(String prefix) {
            return GLOBAL_STORE_NAME;
        }
    }, "store-");
    builder.addGlobalStore((StoreBuilder) new KeyValueStoreMaterializer<>(materialized).materialize().withLoggingDisabled(), "sourceName", null, null, null, GLOBAL_STORE_TOPIC_NAME, "processorName", new KTableSource<>(GLOBAL_STORE_NAME));
    final HashMap<String, Object> properties = new HashMap<>();
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "blah");
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "blah");
    properties.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    config = new StreamsConfig(properties);
    globalStreamThread = new GlobalStreamThread(builder.buildGlobalStateTopology(), config, mockConsumer, new StateDirectory(config, time), 0, new Metrics(), new MockTime(), "clientId", stateRestoreListener);
}