Examples with SessionStore org.apache.kafka.streams.state.SessionStore used on opensource projects

Example 6 with SessionStore

use of org.apache.kafka.streams.state.SessionStore in project kafka by apache.

the class SuppressScenarioTest method shouldSupportFinalResultsForSessionWindows.

@Test
public void shouldSupportFinalResultsForSessionWindows() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Windowed<String>, Long> valueCounts = builder.stream("input", Consumed.with(STRING_SERDE, STRING_SERDE)).groupBy((String k, String v) -> k, Grouped.with(STRING_SERDE, STRING_SERDE)).windowedBy(SessionWindows.with(ofMillis(5L)).grace(ofMillis(0L))).count(Materialized.<String, Long, SessionStore<Bytes, byte[]>>as("counts").withCachingDisabled());
    valueCounts.suppress(untilWindowCloses(unbounded())).toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    System.out.println(topology.describe());
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        // first window
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v1", 5L);
        // arbitrarily disordered records are admitted, because the *window* is not closed until stream-time > window-end + grace
        inputTopic.pipeInput("k1", "v1", 1L);
        // any record in the same partition advances stream time (note the key is different)
        inputTopic.pipeInput("k2", "v1", 11L);
        // late event for first window - this should get dropped from all streams, since the first window is now closed.
        inputTopic.pipeInput("k1", "v1", 5L);
        // just pushing stream time forward to flush the other events through.
        inputTopic.pipeInput("k1", "v1", 30L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("[k1@0/0]", 1L, 0L), new KeyValueTimestamp<>("[k1@0/0]", null, 0L), new KeyValueTimestamp<>("[k1@0/5]", 2L, 5L), new KeyValueTimestamp<>("[k1@0/5]", null, 5L), new KeyValueTimestamp<>("[k1@0/5]", 3L, 5L), new KeyValueTimestamp<>("[k2@11/11]", 1L, 11L), new KeyValueTimestamp<>("[k1@30/30]", 1L, 30L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("[k1@0/5]", 3L, 5L), new KeyValueTimestamp<>("[k2@11/11]", 1L, 11L)));
    }
}

Example 7 with SessionStore

use of org.apache.kafka.streams.state.SessionStore in project kafka by apache.

the class SuppressTopologyTest method shouldUseNumberingForAnonymousFinalSuppressionNode.

@Test
public void shouldUseNumberingForAnonymousFinalSuppressionNode() {
    final StreamsBuilder anonymousNodeBuilder = new StreamsBuilder();
    anonymousNodeBuilder.stream("input", Consumed.with(STRING_SERDE, STRING_SERDE)).groupBy((String k, String v) -> k, Grouped.with(STRING_SERDE, STRING_SERDE)).windowedBy(SessionWindows.with(ofMillis(5L)).grace(ofMillis(5L))).count(Materialized.<String, Long, SessionStore<Bytes, byte[]>>as("counts").withCachingDisabled()).suppress(untilWindowCloses(unbounded())).toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    final String anonymousNodeTopology = anonymousNodeBuilder.build().describe().toString();
    // without the name, the suppression node increments the topology index
    assertThat(anonymousNodeTopology, is(ANONYMOUS_FINAL_TOPOLOGY));
}

Example 8 with SessionStore

use of org.apache.kafka.streams.state.SessionStore in project kafka by apache.

the class SuppressTopologyTest method shouldApplyNameToFinalSuppressionNode.

@Test
public void shouldApplyNameToFinalSuppressionNode() {
    final StreamsBuilder namedNodeBuilder = new StreamsBuilder();
    namedNodeBuilder.stream("input", Consumed.with(STRING_SERDE, STRING_SERDE)).groupBy((String k, String v) -> k, Grouped.with(STRING_SERDE, STRING_SERDE)).windowedBy(SessionWindows.with(ofMillis(5L)).grace(ofMillis(5L))).count(Materialized.<String, Long, SessionStore<Bytes, byte[]>>as("counts").withCachingDisabled()).suppress(untilWindowCloses(unbounded()).withName("myname")).toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    final String namedNodeTopology = namedNodeBuilder.build().describe().toString();
    // without the name, the suppression node does not increment the topology index
    assertThat(namedNodeTopology, is(NAMED_FINAL_TOPOLOGY));
}

Example 9 with SessionStore

use of org.apache.kafka.streams.state.SessionStore in project kafka-streams-examples by confluentinc.

the class SessionWindowsExample method createStreams.

static KafkaStreams createStreams(final String bootstrapServers, final String schemaRegistryUrl, final String stateDir) {
    final Properties config = new Properties();
    // Give the Streams application a unique name.  The name must be unique in the Kafka cluster
    // against which the application is run.
    config.put(StreamsConfig.APPLICATION_ID_CONFIG, "session-windows-example");
    config.put(StreamsConfig.CLIENT_ID_CONFIG, "session-windows-example-client");
    // Where to find Kafka broker(s).
    config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.put(StreamsConfig.STATE_DIR_CONFIG, stateDir);
    // Set to earliest so we don't miss any data that arrived in the topics before the process
    // started
    config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    // disable caching to see session merging
    config.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // create and configure the SpecificAvroSerdes required in this example
    final SpecificAvroSerde<PlayEvent> playEventSerde = new SpecificAvroSerde<>();
    final Map<String, String> serdeConfig = Collections.singletonMap(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, schemaRegistryUrl);
    playEventSerde.configure(serdeConfig, false);
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(PLAY_EVENTS, Consumed.with(Serdes.String(), playEventSerde)).groupByKey(Serialized.with(Serdes.String(), playEventSerde)).windowedBy(SessionWindows.with(INACTIVITY_GAP)).count(Materialized.<String, Long, SessionStore<Bytes, byte[]>>as(PLAY_EVENTS_PER_SESSION).withKeySerde(Serdes.String()).withValueSerde(Serdes.Long())).toStream().map((key, value) -> new KeyValue<>(key.key() + "@" + key.window().start() + "->" + key.window().end(), value)).to(PLAY_EVENTS_PER_SESSION, Produced.with(Serdes.String(), Serdes.Long()));
    return new KafkaStreams(builder.build(), new StreamsConfig(config));
}

Example 10 with SessionStore

use of org.apache.kafka.streams.state.SessionStore in project kafka by apache.

the class SessionWindowedKStreamImpl method materialize.

private <VR> StoreBuilder<SessionStore<K, VR>> materialize(final MaterializedInternal<K, VR, SessionStore<Bytes, byte[]>> materialized) {
    SessionBytesStoreSupplier supplier = (SessionBytesStoreSupplier) materialized.storeSupplier();
    if (supplier == null) {
        final long retentionPeriod = materialized.retention() != null ? materialized.retention().toMillis() : windows.inactivityGap() + windows.gracePeriodMs();
        if ((windows.inactivityGap() + windows.gracePeriodMs()) > retentionPeriod) {
            throw new IllegalArgumentException("The retention period of the session store " + materialized.storeName() + " must be no smaller than the session inactivity gap plus the" + " grace period." + " Got gap=[" + windows.inactivityGap() + "]," + " grace=[" + windows.gracePeriodMs() + "]," + " retention=[" + retentionPeriod + "]");
        }
        supplier = Stores.persistentSessionStore(materialized.storeName(), Duration.ofMillis(retentionPeriod));
    }
    final StoreBuilder<SessionStore<K, VR>> builder = Stores.sessionStoreBuilder(supplier, materialized.keySerde(), materialized.valueSerde());
    if (materialized.loggingEnabled()) {
        builder.withLoggingEnabled(materialized.logConfig());
    } else {
        builder.withLoggingDisabled();
    }
    if (materialized.cachingEnabled()) {
        builder.withCachingEnabled();
    }
    return builder;
}