Examples with WindowBytesStoreSupplier org.apache.kafka.streams.state.WindowBytesStoreSupplier used on opensource projects

Example 6 with WindowBytesStoreSupplier

use of org.apache.kafka.streams.state.WindowBytesStoreSupplier in project kafka by apache.

the class KStreamKStreamOuterJoinTest method testOuterJoinWithInMemoryCustomSuppliers.

@Test
public void testOuterJoinWithInMemoryCustomSuppliers() {
    final JoinWindows joinWindows = JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100L));
    final WindowBytesStoreSupplier thisStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    final WindowBytesStoreSupplier otherStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store-other", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    final StreamJoined<Integer, String, String> streamJoined = StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String());
    runOuterJoin(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier), joinWindows);
}

Example 7 with WindowBytesStoreSupplier

use of org.apache.kafka.streams.state.WindowBytesStoreSupplier in project kafka by apache.

the class TopologyTest method streamStreamJoinTopologyWithCustomStoresSuppliers.

@Test
public void streamStreamJoinTopologyWithCustomStoresSuppliers() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, String> stream1;
    final KStream<Integer, String> stream2;
    stream1 = builder.stream("input-topic1");
    stream2 = builder.stream("input-topic2");
    final JoinWindows joinWindows = JoinWindows.of(ofMillis(100));
    final WindowBytesStoreSupplier thisStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    final WindowBytesStoreSupplier otherStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store-other", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    stream1.join(stream2, MockValueJoiner.TOSTRING_JOINER, joinWindows, StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()).withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier));
    final TopologyDescription describe = builder.build().describe();
    assertEquals("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input-topic1])\n" + "      --> KSTREAM-WINDOWED-0000000002\n" + "    Source: KSTREAM-SOURCE-0000000001 (topics: [input-topic2])\n" + "      --> KSTREAM-WINDOWED-0000000003\n" + "    Processor: KSTREAM-WINDOWED-0000000002 (stores: [in-memory-join-store])\n" + "      --> KSTREAM-JOINTHIS-0000000004\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: KSTREAM-WINDOWED-0000000003 (stores: [in-memory-join-store-other])\n" + "      --> KSTREAM-JOINOTHER-0000000005\n" + "      <-- KSTREAM-SOURCE-0000000001\n" + "    Processor: KSTREAM-JOINOTHER-0000000005 (stores: [in-memory-join-store])\n" + "      --> KSTREAM-MERGE-0000000006\n" + "      <-- KSTREAM-WINDOWED-0000000003\n" + "    Processor: KSTREAM-JOINTHIS-0000000004 (stores: [in-memory-join-store-other])\n" + "      --> KSTREAM-MERGE-0000000006\n" + "      <-- KSTREAM-WINDOWED-0000000002\n" + "    Processor: KSTREAM-MERGE-0000000006 (stores: [])\n" + "      --> none\n" + "      <-- KSTREAM-JOINTHIS-0000000004, KSTREAM-JOINOTHER-0000000005\n\n", describe.toString());
}

Example 8 with WindowBytesStoreSupplier

use of org.apache.kafka.streams.state.WindowBytesStoreSupplier in project kafka by apache.

the class IQv2StoreIntegrationTest method beforeTest.

@Before
public void beforeTest() {
    final StoreSupplier<?> supplier = storeToTest.supplier();
    final Properties streamsConfig = streamsConfiguration(cache, log, storeToTest.name(), kind);
    final StreamsBuilder builder = new StreamsBuilder();
    if (Objects.equals(kind, "DSL") && supplier instanceof KeyValueBytesStoreSupplier) {
        setUpKeyValueDSLTopology((KeyValueBytesStoreSupplier) supplier, builder);
    } else if (Objects.equals(kind, "PAPI") && supplier instanceof KeyValueBytesStoreSupplier) {
        setUpKeyValuePAPITopology((KeyValueBytesStoreSupplier) supplier, builder);
    } else if (Objects.equals(kind, "DSL") && supplier instanceof WindowBytesStoreSupplier) {
        setUpWindowDSLTopology((WindowBytesStoreSupplier) supplier, builder);
    } else if (Objects.equals(kind, "PAPI") && supplier instanceof WindowBytesStoreSupplier) {
        setUpWindowPAPITopology((WindowBytesStoreSupplier) supplier, builder);
    } else if (Objects.equals(kind, "DSL") && supplier instanceof SessionBytesStoreSupplier) {
        setUpSessionDSLTopology((SessionBytesStoreSupplier) supplier, builder);
    } else if (Objects.equals(kind, "PAPI") && supplier instanceof SessionBytesStoreSupplier) {
        setUpSessionPAPITopology((SessionBytesStoreSupplier) supplier, builder);
    } else {
        throw new AssertionError("Store supplier is an unrecognized type.");
    }
    // Don't need to wait for running, since tests can use iqv2 to wait until they
    // get a valid response.
    kafkaStreams = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
}

Example 9 with WindowBytesStoreSupplier

use of org.apache.kafka.streams.state.WindowBytesStoreSupplier in project kafka by apache.

the class WindowStoreFetchTest method getStoreConfig.

private Materialized<String, Long, WindowStore<Bytes, byte[]>> getStoreConfig(final StoreType type, final String name, final boolean cachingEnabled, final boolean loggingEnabled) {
    final Supplier<WindowBytesStoreSupplier> createStore = () -> {
        if (type == StoreType.InMemory) {
            return Stores.inMemoryWindowStore(STORE_NAME, Duration.ofMillis(RETENTION_MS), Duration.ofMillis(WINDOW_SIZE), false);
        } else if (type == StoreType.RocksDB) {
            return Stores.persistentWindowStore(STORE_NAME, Duration.ofMillis(RETENTION_MS), Duration.ofMillis(WINDOW_SIZE), false);
        } else if (type == StoreType.Timed) {
            return Stores.persistentTimestampedWindowStore(STORE_NAME, Duration.ofMillis(RETENTION_MS), Duration.ofMillis(WINDOW_SIZE), false);
        } else {
            return Stores.inMemoryWindowStore(STORE_NAME, Duration.ofMillis(RETENTION_MS), Duration.ofMillis(WINDOW_SIZE), false);
        }
    };
    final WindowBytesStoreSupplier stateStoreSupplier = createStore.get();
    final Materialized<String, Long, WindowStore<Bytes, byte[]>> stateStoreConfig = Materialized.<String, Long>as(stateStoreSupplier).withKeySerde(Serdes.String()).withValueSerde(Serdes.Long());
    if (cachingEnabled) {
        stateStoreConfig.withCachingEnabled();
    } else {
        stateStoreConfig.withCachingDisabled();
    }
    if (loggingEnabled) {
        stateStoreConfig.withLoggingEnabled(new HashMap<String, String>());
    } else {
        stateStoreConfig.withLoggingDisabled();
    }
    return stateStoreConfig;
}

Example 10 with WindowBytesStoreSupplier

use of org.apache.kafka.streams.state.WindowBytesStoreSupplier in project kafka by apache.

the class TimeWindowedKStreamImpl method materialize.

private <VR> StoreBuilder<TimestampedWindowStore<K, VR>> materialize(final MaterializedInternal<K, VR, WindowStore<Bytes, byte[]>> materialized) {
    WindowBytesStoreSupplier supplier = (WindowBytesStoreSupplier) materialized.storeSupplier();
    if (supplier == null) {
        final long retentionPeriod = materialized.retention() != null ? materialized.retention().toMillis() : windows.size() + windows.gracePeriodMs();
        if ((windows.size() + windows.gracePeriodMs()) > retentionPeriod) {
            throw new IllegalArgumentException("The retention period of the window store " + name + " must be no smaller than its window size plus the grace period." + " Got size=[" + windows.size() + "]," + " grace=[" + windows.gracePeriodMs() + "]," + " retention=[" + retentionPeriod + "]");
        }
        supplier = Stores.persistentTimestampedWindowStore(materialized.storeName(), Duration.ofMillis(retentionPeriod), Duration.ofMillis(windows.size()), false);
    }
    final StoreBuilder<TimestampedWindowStore<K, VR>> builder = Stores.timestampedWindowStoreBuilder(supplier, materialized.keySerde(), materialized.valueSerde());
    if (materialized.loggingEnabled()) {
        builder.withLoggingEnabled(materialized.logConfig());
    } else {
        builder.withLoggingDisabled();
    }
    if (materialized.cachingEnabled()) {
        builder.withCachingEnabled();
    }
    return builder;
}