Examples with MaterializedInternal org.apache.kafka.streams.kstream.internals.MaterializedInternal used on opensource projects

Example 11 with MaterializedInternal

use of org.apache.kafka.streams.kstream.internals.MaterializedInternal in project apache-kafka-on-k8s by banzaicloud.

the class GlobalStreamThreadTest method before.

@SuppressWarnings("unchecked")
@Before
public void before() {
    final MaterializedInternal<Object, Object, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<Object, Object, KeyValueStore<Bytes, byte[]>>with(null, null), new InternalNameProvider() {

        @Override
        public String newProcessorName(String prefix) {
            return "processorName";
        }

        @Override
        public String newStoreName(String prefix) {
            return GLOBAL_STORE_NAME;
        }
    }, "store-");
    builder.addGlobalStore((StoreBuilder) new KeyValueStoreMaterializer<>(materialized).materialize().withLoggingDisabled(), "sourceName", null, null, null, GLOBAL_STORE_TOPIC_NAME, "processorName", new KTableSource<>(GLOBAL_STORE_NAME));
    final HashMap<String, Object> properties = new HashMap<>();
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "blah");
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "blah");
    properties.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    config = new StreamsConfig(properties);
    globalStreamThread = new GlobalStreamThread(builder.buildGlobalStateTopology(), config, mockConsumer, new StateDirectory(config, time), 0, new Metrics(), new MockTime(), "clientId", stateRestoreListener);
}

Example 12 with MaterializedInternal

use of org.apache.kafka.streams.kstream.internals.MaterializedInternal in project apache-kafka-on-k8s by banzaicloud.

the class KeyValueStoreMaterializerTest method shouldCreateBuilderThatBuildsStoreWithCachingDisabled.

@Test
public void shouldCreateBuilderThatBuildsStoreWithCachingDisabled() {
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("store").withCachingDisabled(), nameProvider, storePrefix);
    final KeyValueStoreMaterializer<String, String> materializer = new KeyValueStoreMaterializer<>(materialized);
    final StoreBuilder<KeyValueStore<String, String>> builder = materializer.materialize();
    final KeyValueStore<String, String> store = builder.build();
    final WrappedStateStore logging = (WrappedStateStore) ((WrappedStateStore) store).wrappedStore();
    assertThat(logging, instanceOf(ChangeLoggingKeyValueBytesStore.class));
}

Example 13 with MaterializedInternal

use of org.apache.kafka.streams.kstream.internals.MaterializedInternal in project apache-kafka-on-k8s by banzaicloud.

the class KeyValueStoreMaterializerTest method shouldCreateBuilderThatBuildsMeteredStoreWithCachingAndLoggingEnabled.

@Test
public void shouldCreateBuilderThatBuildsMeteredStoreWithCachingAndLoggingEnabled() {
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("store"), nameProvider, storePrefix);
    final KeyValueStoreMaterializer<String, String> materializer = new KeyValueStoreMaterializer<>(materialized);
    final StoreBuilder<KeyValueStore<String, String>> builder = materializer.materialize();
    final KeyValueStore<String, String> store = builder.build();
    final WrappedStateStore caching = (WrappedStateStore) ((WrappedStateStore) store).wrappedStore();
    final StateStore logging = caching.wrappedStore();
    assertThat(store, instanceOf(MeteredKeyValueBytesStore.class));
    assertThat(caching, instanceOf(CachedStateStore.class));
    assertThat(logging, instanceOf(ChangeLoggingKeyValueBytesStore.class));
}

Example 14 with MaterializedInternal

use of org.apache.kafka.streams.kstream.internals.MaterializedInternal in project apache-kafka-on-k8s by banzaicloud.

the class KeyValueStoreMaterializerTest method shouldCreateBuilderThatBuildsStoreWithCachingAndLoggingDisabled.

@Test
public void shouldCreateBuilderThatBuildsStoreWithCachingAndLoggingDisabled() {
    final MaterializedInternal<String, String, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("store").withCachingDisabled().withLoggingDisabled(), nameProvider, storePrefix);
    final KeyValueStoreMaterializer<String, String> materializer = new KeyValueStoreMaterializer<>(materialized);
    final StoreBuilder<KeyValueStore<String, String>> builder = materializer.materialize();
    final KeyValueStore<String, String> store = builder.build();
    final StateStore wrapped = ((WrappedStateStore) store).wrappedStore();
    assertThat(wrapped, not(instanceOf(CachedStateStore.class)));
    assertThat(wrapped, not(instanceOf(ChangeLoggingKeyValueBytesStore.class)));
}

Example 15 with MaterializedInternal

use of org.apache.kafka.streams.kstream.internals.MaterializedInternal in project kafka by apache.

the class StreamThreadTest method shouldUpdateStandbyTask.

@SuppressWarnings("unchecked")
@Test
public void shouldUpdateStandbyTask() throws Exception {
    final String storeName1 = "count-one";
    final String storeName2 = "table-two";
    final String changelogName1 = APPLICATION_ID + "-" + storeName1 + "-changelog";
    final String changelogName2 = APPLICATION_ID + "-" + storeName2 + "-changelog";
    final TopicPartition partition1 = new TopicPartition(changelogName1, 1);
    final TopicPartition partition2 = new TopicPartition(changelogName2, 1);
    internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).groupByKey().count(Materialized.as(storeName1));
    final MaterializedInternal<Object, Object, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.as(storeName2), internalStreamsBuilder, "");
    internalStreamsBuilder.table(topic2, new ConsumedInternal<>(), materialized);
    internalStreamsBuilder.buildAndOptimizeTopology();
    final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
    final MockConsumer<byte[], byte[]> restoreConsumer = clientSupplier.restoreConsumer;
    restoreConsumer.updatePartitions(changelogName1, Collections.singletonList(new PartitionInfo(changelogName1, 1, null, new Node[0], new Node[0])));
    restoreConsumer.updateEndOffsets(Collections.singletonMap(partition1, 10L));
    restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition1, 0L));
    restoreConsumer.updateEndOffsets(Collections.singletonMap(partition2, 10L));
    restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition2, 0L));
    final OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(task3), CHECKPOINT_FILE_NAME));
    checkpoint.write(Collections.singletonMap(partition2, 5L));
    thread.setState(StreamThread.State.STARTING);
    thread.rebalanceListener().onPartitionsRevoked(Collections.emptySet());
    final Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    // assign single partition
    standbyTasks.put(task1, Collections.singleton(t1p1));
    standbyTasks.put(task3, Collections.singleton(t2p1));
    thread.taskManager().handleAssignment(emptyMap(), standbyTasks);
    thread.taskManager().tryToCompleteRestoration(mockTime.milliseconds(), null);
    thread.rebalanceListener().onPartitionsAssigned(Collections.emptyList());
    thread.runOnce();
    final StandbyTask standbyTask1 = standbyTask(thread.taskManager(), t1p1);
    final StandbyTask standbyTask2 = standbyTask(thread.taskManager(), t2p1);
    assertEquals(task1, standbyTask1.id());
    assertEquals(task3, standbyTask2.id());
    final KeyValueStore<Object, Long> store1 = (KeyValueStore<Object, Long>) standbyTask1.getStore(storeName1);
    final KeyValueStore<Object, Long> store2 = (KeyValueStore<Object, Long>) standbyTask2.getStore(storeName2);
    assertEquals(0L, store1.approximateNumEntries());
    assertEquals(0L, store2.approximateNumEntries());
    // let the store1 be restored from 0 to 10; store2 be restored from 5 (checkpointed) to 10
    for (long i = 0L; i < 10L; i++) {
        restoreConsumer.addRecord(new ConsumerRecord<>(changelogName1, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes()));
        restoreConsumer.addRecord(new ConsumerRecord<>(changelogName2, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes()));
    }
    thread.runOnce();
    assertEquals(10L, store1.approximateNumEntries());
    assertEquals(4L, store2.approximateNumEntries());
    thread.taskManager().shutdown(true);
}