Examples with Windowed org.apache.kafka.streams.kstream.Windowed used on opensource projects

Example 21 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class RocksDBSessionStoreTest method shouldFindSessionsToMerge.

@Test
public void shouldFindSessionsToMerge() throws Exception {
    final Windowed<String> session1 = new Windowed<>("a", new SessionWindow(0, 100));
    final Windowed<String> session2 = new Windowed<>("a", new SessionWindow(101, 200));
    final Windowed<String> session3 = new Windowed<>("a", new SessionWindow(201, 300));
    final Windowed<String> session4 = new Windowed<>("a", new SessionWindow(301, 400));
    final Windowed<String> session5 = new Windowed<>("a", new SessionWindow(401, 500));
    sessionStore.put(session1, 1L);
    sessionStore.put(session2, 2L);
    sessionStore.put(session3, 3L);
    sessionStore.put(session4, 4L);
    sessionStore.put(session5, 5L);
    final KeyValueIterator<Windowed<String>, Long> results = sessionStore.findSessions("a", 150, 300);
    assertEquals(session2, results.next().key);
    assertEquals(session3, results.next().key);
    assertFalse(results.hasNext());
}

Example 22 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class RocksDBSessionStoreTest method shouldPutAndFindSessionsInRange.

@Test
public void shouldPutAndFindSessionsInRange() throws Exception {
    final String key = "a";
    final Windowed<String> a1 = new Windowed<>(key, new SessionWindow(10, 10L));
    final Windowed<String> a2 = new Windowed<>(key, new SessionWindow(500L, 1000L));
    sessionStore.put(a1, 1L);
    sessionStore.put(a2, 2L);
    sessionStore.put(new Windowed<>(key, new SessionWindow(1500L, 2000L)), 1L);
    sessionStore.put(new Windowed<>(key, new SessionWindow(2500L, 3000L)), 2L);
    final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L));
    final KeyValueIterator<Windowed<String>, Long> values = sessionStore.findSessions(key, 0, 1000L);
    assertEquals(expected, toList(values));
}

Example 23 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class PageViewTypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    KStreamBuilder builder = new KStreamBuilder();
    // TODO: the following can be removed with a serialization factory
    Map<String, Object> serdeProps = new HashMap<>();
    final Serializer<PageView> pageViewSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewSerializer.configure(serdeProps, false);
    final Deserializer<PageView> pageViewDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewDeserializer.configure(serdeProps, false);
    final Serde<PageView> pageViewSerde = Serdes.serdeFrom(pageViewSerializer, pageViewDeserializer);
    final Serializer<UserProfile> userProfileSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileSerializer.configure(serdeProps, false);
    final Deserializer<UserProfile> userProfileDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileDeserializer.configure(serdeProps, false);
    final Serde<UserProfile> userProfileSerde = Serdes.serdeFrom(userProfileSerializer, userProfileDeserializer);
    final Serializer<WindowedPageViewByRegion> wPageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<WindowedPageViewByRegion> wPageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<WindowedPageViewByRegion> wPageViewByRegionSerde = Serdes.serdeFrom(wPageViewByRegionSerializer, wPageViewByRegionDeserializer);
    final Serializer<RegionCount> regionCountSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountSerializer.configure(serdeProps, false);
    final Deserializer<RegionCount> regionCountDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountDeserializer.configure(serdeProps, false);
    final Serde<RegionCount> regionCountSerde = Serdes.serdeFrom(regionCountSerializer, regionCountDeserializer);
    final Serializer<PageViewByRegion> pageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<PageViewByRegion> pageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<PageViewByRegion> pageViewByRegionSerde = Serdes.serdeFrom(pageViewByRegionSerializer, pageViewByRegionDeserializer);
    KStream<String, PageView> views = builder.stream(Serdes.String(), pageViewSerde, "streams-pageview-input");
    KTable<String, UserProfile> users = builder.table(Serdes.String(), userProfileSerde, "streams-userprofile-input", "streams-userprofile-store-name");
    KStream<WindowedPageViewByRegion, RegionCount> regionCount = views.leftJoin(users, new ValueJoiner<PageView, UserProfile, PageViewByRegion>() {

        @Override
        public PageViewByRegion apply(PageView view, UserProfile profile) {
            PageViewByRegion viewByRegion = new PageViewByRegion();
            viewByRegion.user = view.user;
            viewByRegion.page = view.page;
            if (profile != null) {
                viewByRegion.region = profile.region;
            } else {
                viewByRegion.region = "UNKNOWN";
            }
            return viewByRegion;
        }
    }).map(new KeyValueMapper<String, PageViewByRegion, KeyValue<String, PageViewByRegion>>() {

        @Override
        public KeyValue<String, PageViewByRegion> apply(String user, PageViewByRegion viewRegion) {
            return new KeyValue<>(viewRegion.region, viewRegion);
        }
    }).groupByKey(Serdes.String(), pageViewByRegionSerde).count(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000), "RollingSevenDaysOfPageViewsByRegion").toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<WindowedPageViewByRegion, RegionCount>>() {

        @Override
        public KeyValue<WindowedPageViewByRegion, RegionCount> apply(Windowed<String> key, Long value) {
            WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
            wViewByRegion.windowStart = key.window().start();
            wViewByRegion.region = key.key();
            RegionCount rCount = new RegionCount();
            rCount.region = key.key();
            rCount.count = value;
            return new KeyValue<>(wViewByRegion, rCount);
        }
    });
    // write to the result topic
    regionCount.to(wPageViewByRegionSerde, regionCountSerde, "streams-pageviewstats-typed-output");
    KafkaStreams streams = new KafkaStreams(builder, props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 24 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class PageViewUntypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-untyped");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    KStreamBuilder builder = new KStreamBuilder();
    final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
    final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
    final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);
    KStream<String, JsonNode> views = builder.stream(Serdes.String(), jsonSerde, "streams-pageview-input");
    KTable<String, JsonNode> users = builder.table(Serdes.String(), jsonSerde, "streams-userprofile-input", "streams-userprofile-store-name");
    KTable<String, String> userRegions = users.mapValues(new ValueMapper<JsonNode, String>() {

        @Override
        public String apply(JsonNode record) {
            return record.get("region").textValue();
        }
    });
    KStream<JsonNode, JsonNode> regionCount = views.leftJoin(userRegions, new ValueJoiner<JsonNode, String, JsonNode>() {

        @Override
        public JsonNode apply(JsonNode view, String region) {
            ObjectNode jNode = JsonNodeFactory.instance.objectNode();
            return jNode.put("user", view.get("user").textValue()).put("page", view.get("page").textValue()).put("region", region == null ? "UNKNOWN" : region);
        }
    }).map(new KeyValueMapper<String, JsonNode, KeyValue<String, JsonNode>>() {

        @Override
        public KeyValue<String, JsonNode> apply(String user, JsonNode viewRegion) {
            return new KeyValue<>(viewRegion.get("region").textValue(), viewRegion);
        }
    }).groupByKey(Serdes.String(), jsonSerde).count(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000), "RollingSevenDaysOfPageViewsByRegion").toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<JsonNode, JsonNode>>() {

        @Override
        public KeyValue<JsonNode, JsonNode> apply(Windowed<String> key, Long value) {
            ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
            keyNode.put("window-start", key.window().start()).put("region", key.key());
            ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
            valueNode.put("count", value);
            return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
        }
    });
    // write to the result topic
    regionCount.to(jsonSerde, jsonSerde, "streams-pageviewstats-untyped-output");
    KafkaStreams streams = new KafkaStreams(builder, props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 25 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KStreamWindowAggregateTest method testJoin.

@Test
public void testJoin() throws Exception {
    final File baseDir = Files.createTempDirectory("test").toFile();
    try {
        final KStreamBuilder builder = new KStreamBuilder();
        String topic1 = "topic1";
        String topic2 = "topic2";
        KStream<String, String> stream1 = builder.stream(strSerde, strSerde, topic1);
        KTable<Windowed<String>, String> table1 = stream1.groupByKey(strSerde, strSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic1-Canonized");
        MockProcessorSupplier<Windowed<String>, String> proc1 = new MockProcessorSupplier<>();
        table1.toStream().process(proc1);
        KStream<String, String> stream2 = builder.stream(strSerde, strSerde, topic2);
        KTable<Windowed<String>, String> table2 = stream2.groupByKey(strSerde, strSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic2-Canonized");
        MockProcessorSupplier<Windowed<String>, String> proc2 = new MockProcessorSupplier<>();
        table2.toStream().process(proc2);
        MockProcessorSupplier<Windowed<String>, String> proc3 = new MockProcessorSupplier<>();
        table1.join(table2, new ValueJoiner<String, String, String>() {

            @Override
            public String apply(String p1, String p2) {
                return p1 + "%" + p2;
            }
        }).toStream().process(proc3);
        driver = new KStreamTestDriver(builder, baseDir);
        setRecordContext(0, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        setRecordContext(1, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(2, topic1);
        driver.process(topic1, "C", "3");
        driver.flushState();
        setRecordContext(3, topic1);
        driver.process(topic1, "D", "4");
        driver.flushState();
        setRecordContext(4, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        proc1.checkAndClearProcessResult("[A@0]:0+1", "[B@0]:0+2", "[C@0]:0+3", "[D@0]:0+4", "[A@0]:0+1+1");
        proc2.checkAndClearProcessResult();
        proc3.checkAndClearProcessResult();
        setRecordContext(5, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        setRecordContext(6, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(7, topic1);
        driver.process(topic1, "D", "4");
        driver.flushState();
        setRecordContext(8, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(9, topic1);
        driver.process(topic1, "C", "3");
        driver.flushState();
        proc1.checkAndClearProcessResult("[A@0]:0+1+1+1", "[A@5]:0+1", "[B@0]:0+2+2", "[B@5]:0+2", "[D@0]:0+4+4", "[D@5]:0+4", "[B@0]:0+2+2+2", "[B@5]:0+2+2", "[C@0]:0+3+3", "[C@5]:0+3");
        proc2.checkAndClearProcessResult();
        proc3.checkAndClearProcessResult();
        setRecordContext(0, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        setRecordContext(1, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(2, topic1);
        driver.process(topic2, "C", "c");
        driver.flushState();
        setRecordContext(3, topic1);
        driver.process(topic2, "D", "d");
        driver.flushState();
        setRecordContext(4, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        proc1.checkAndClearProcessResult();
        proc2.checkAndClearProcessResult("[A@0]:0+a", "[B@0]:0+b", "[C@0]:0+c", "[D@0]:0+d", "[A@0]:0+a+a");
        proc3.checkAndClearProcessResult("[A@0]:0+1+1+1%0+a", "[B@0]:0+2+2+2%0+b", "[C@0]:0+3+3%0+c", "[D@0]:0+4+4%0+d", "[A@0]:0+1+1+1%0+a+a");
        setRecordContext(5, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        setRecordContext(6, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(7, topic1);
        driver.process(topic2, "D", "d");
        driver.flushState();
        setRecordContext(8, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(9, topic1);
        driver.process(topic2, "C", "c");
        driver.flushState();
        proc1.checkAndClearProcessResult();
        proc2.checkAndClearProcessResult("[A@0]:0+a+a+a", "[A@5]:0+a", "[B@0]:0+b+b", "[B@5]:0+b", "[D@0]:0+d+d", "[D@5]:0+d", "[B@0]:0+b+b+b", "[B@5]:0+b+b", "[C@0]:0+c+c", "[C@5]:0+c");
        proc3.checkAndClearProcessResult("[A@0]:0+1+1+1%0+a+a+a", "[A@5]:0+1%0+a", "[B@0]:0+2+2+2%0+b+b", "[B@5]:0+2+2%0+b", "[D@0]:0+4+4%0+d+d", "[D@5]:0+4%0+d", "[B@0]:0+2+2+2%0+b+b+b", "[B@5]:0+2+2%0+b+b", "[C@0]:0+3+3%0+c+c", "[C@5]:0+3%0+c");
    } finally {
        Utils.delete(baseDir);
    }
}