Examples with Windowed org.apache.kafka.streams.kstream.Windowed used on opensource projects

Example 26 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldGroupByKey.

@Test
public void shouldGroupByKey() throws Exception {
    final long timestamp = mockTime.milliseconds();
    produceMessages(timestamp);
    produceMessages(timestamp);
    stream.groupByKey(Serdes.Integer(), Serdes.String()).count(TimeWindows.of(500L), "count-windows").toStream(new KeyValueMapper<Windowed<Integer>, Long, String>() {

        @Override
        public String apply(final Windowed<Integer> windowedKey, final Long value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.Long(), outputTopic);
    startStreams();
    final List<KeyValue<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 10);
    Collections.sort(results, new Comparator<KeyValue<String, Long>>() {

        @Override
        public int compare(final KeyValue<String, Long> o1, final KeyValue<String, Long> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    });
    final long window = timestamp / 500 * 500;
    assertThat(results, is(Arrays.asList(KeyValue.pair("1@" + window, 1L), KeyValue.pair("1@" + window, 2L), KeyValue.pair("2@" + window, 1L), KeyValue.pair("2@" + window, 2L), KeyValue.pair("3@" + window, 1L), KeyValue.pair("3@" + window, 2L), KeyValue.pair("4@" + window, 1L), KeyValue.pair("4@" + window, 2L), KeyValue.pair("5@" + window, 1L), KeyValue.pair("5@" + window, 2L))));
}

Example 27 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceWindowed.

@Test
public void shouldReduceWindowed() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = mockTime.milliseconds();
    produceMessages(secondBatchTimestamp);
    produceMessages(secondBatchTimestamp);
    groupedStream.reduce(reducer, TimeWindows.of(500L), "reduce-time-windows").toStream(new KeyValueMapper<Windowed<String>, String, String>() {

        @Override
        public String apply(final Windowed<String> windowedKey, final String value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.String(), outputTopic);
    startStreams();
    final List<KeyValue<String, String>> windowedOutput = receiveMessages(new StringDeserializer(), new StringDeserializer(), 15);
    final Comparator<KeyValue<String, String>> comparator = new Comparator<KeyValue<String, String>>() {

        @Override
        public int compare(final KeyValue<String, String> o1, final KeyValue<String, String> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedOutput, comparator);
    final long firstBatchWindow = firstBatchTimestamp / 500 * 500;
    final long secondBatchWindow = secondBatchTimestamp / 500 * 500;
    assertThat(windowedOutput, is(Arrays.asList(new KeyValue<>("A@" + firstBatchWindow, "A"), new KeyValue<>("A@" + secondBatchWindow, "A"), new KeyValue<>("A@" + secondBatchWindow, "A:A"), new KeyValue<>("B@" + firstBatchWindow, "B"), new KeyValue<>("B@" + secondBatchWindow, "B"), new KeyValue<>("B@" + secondBatchWindow, "B:B"), new KeyValue<>("C@" + firstBatchWindow, "C"), new KeyValue<>("C@" + secondBatchWindow, "C"), new KeyValue<>("C@" + secondBatchWindow, "C:C"), new KeyValue<>("D@" + firstBatchWindow, "D"), new KeyValue<>("D@" + secondBatchWindow, "D"), new KeyValue<>("D@" + secondBatchWindow, "D:D"), new KeyValue<>("E@" + firstBatchWindow, "E"), new KeyValue<>("E@" + secondBatchWindow, "E"), new KeyValue<>("E@" + secondBatchWindow, "E:E"))));
}

Example 28 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceSessionWindows.

@Test
public void shouldReduceSessionWindows() throws Exception {
    // something to do with time
    final long sessionGap = 1000L;
    final long maintainMillis = sessionGap * 3;
    final long t1 = mockTime.milliseconds();
    final List<KeyValue<String, String>> t1Messages = Arrays.asList(new KeyValue<>("bob", "start"), new KeyValue<>("penny", "start"), new KeyValue<>("jo", "pause"), new KeyValue<>("emily", "pause"));
    IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(userSessionsStream, t1Messages, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), t1);
    final long t2 = t1 + (sessionGap / 2);
    IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(userSessionsStream, Collections.singletonList(new KeyValue<>("emily", "resume")), TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), t2);
    final long t3 = t1 + sessionGap + 1;
    IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(userSessionsStream, Arrays.asList(new KeyValue<>("bob", "pause"), new KeyValue<>("penny", "stop")), TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), t3);
    final long t4 = t3 + (sessionGap / 2);
    IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(userSessionsStream, Arrays.asList(// bobs session continues
    new KeyValue<>("bob", "resume"), // jo's starts new session
    new KeyValue<>("jo", "resume")), TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), t4);
    final Map<Windowed<String>, String> results = new HashMap<>();
    final CountDownLatch latch = new CountDownLatch(11);
    final String userSessionsStore = "UserSessionsStore";
    builder.stream(Serdes.String(), Serdes.String(), userSessionsStream).groupByKey(Serdes.String(), Serdes.String()).reduce(new Reducer<String>() {

        @Override
        public String apply(final String value1, final String value2) {
            return value1 + ":" + value2;
        }
    }, SessionWindows.with(sessionGap).until(maintainMillis), userSessionsStore).foreach(new ForeachAction<Windowed<String>, String>() {

        @Override
        public void apply(final Windowed<String> key, final String value) {
            results.put(key, value);
            latch.countDown();
        }
    });
    startStreams();
    latch.await(30, TimeUnit.SECONDS);
    final ReadOnlySessionStore<String, String> sessionStore = kafkaStreams.store(userSessionsStore, QueryableStoreTypes.<String, String>sessionStore());
    // verify correct data received
    assertThat(results.get(new Windowed<>("bob", new SessionWindow(t1, t1))), equalTo("start"));
    assertThat(results.get(new Windowed<>("penny", new SessionWindow(t1, t1))), equalTo("start"));
    assertThat(results.get(new Windowed<>("jo", new SessionWindow(t1, t1))), equalTo("pause"));
    assertThat(results.get(new Windowed<>("jo", new SessionWindow(t4, t4))), equalTo("resume"));
    assertThat(results.get(new Windowed<>("emily", new SessionWindow(t1, t2))), equalTo("pause:resume"));
    assertThat(results.get(new Windowed<>("bob", new SessionWindow(t3, t4))), equalTo("pause:resume"));
    assertThat(results.get(new Windowed<>("penny", new SessionWindow(t3, t3))), equalTo("stop"));
    // verify can query data via IQ
    final KeyValueIterator<Windowed<String>, String> bob = sessionStore.fetch("bob");
    assertThat(bob.next(), equalTo(KeyValue.pair(new Windowed<>("bob", new SessionWindow(t1, t1)), "start")));
    assertThat(bob.next(), equalTo(KeyValue.pair(new Windowed<>("bob", new SessionWindow(t3, t4)), "pause:resume")));
    assertFalse(bob.hasNext());
}

Example 29 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateWindowed.

@Test
public void shouldAggregateWindowed() throws Exception {
    final long firstTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstTimestamp);
    final long secondTimestamp = mockTime.milliseconds();
    produceMessages(secondTimestamp);
    produceMessages(secondTimestamp);
    groupedStream.aggregate(initializer, aggregator, TimeWindows.of(500L), Serdes.Integer(), "aggregate-by-key-windowed").toStream(new KeyValueMapper<Windowed<String>, Integer, String>() {

        @Override
        public String apply(final Windowed<String> windowedKey, final Integer value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.Integer(), outputTopic);
    startStreams();
    final List<KeyValue<String, Integer>> windowedMessages = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 15);
    final Comparator<KeyValue<String, Integer>> comparator = new Comparator<KeyValue<String, Integer>>() {

        @Override
        public int compare(final KeyValue<String, Integer> o1, final KeyValue<String, Integer> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedMessages, comparator);
    final long firstWindow = firstTimestamp / 500 * 500;
    final long secondWindow = secondTimestamp / 500 * 500;
    assertThat(windowedMessages, is(Arrays.asList(new KeyValue<>("A@" + firstWindow, 1), new KeyValue<>("A@" + secondWindow, 1), new KeyValue<>("A@" + secondWindow, 2), new KeyValue<>("B@" + firstWindow, 1), new KeyValue<>("B@" + secondWindow, 1), new KeyValue<>("B@" + secondWindow, 2), new KeyValue<>("C@" + firstWindow, 1), new KeyValue<>("C@" + secondWindow, 1), new KeyValue<>("C@" + secondWindow, 2), new KeyValue<>("D@" + firstWindow, 1), new KeyValue<>("D@" + secondWindow, 1), new KeyValue<>("D@" + secondWindow, 2), new KeyValue<>("E@" + firstWindow, 1), new KeyValue<>("E@" + secondWindow, 1), new KeyValue<>("E@" + secondWindow, 2))));
}

Example 30 with Windowed

use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.

the class KGroupedStreamImplTest method shouldCountSessionWindows.

@Test
public void shouldCountSessionWindows() throws Exception {
    final Map<Windowed<String>, Long> results = new HashMap<>();
    groupedStream.count(SessionWindows.with(30), "session-store").foreach(new ForeachAction<Windowed<String>, Long>() {

        @Override
        public void apply(final Windowed<String> key, final Long value) {
            results.put(key, value);
        }
    });
    driver = new KStreamTestDriver(builder, TestUtils.tempDirectory());
    driver.setTime(10);
    driver.process(TOPIC, "1", "1");
    driver.setTime(15);
    driver.process(TOPIC, "2", "2");
    driver.setTime(30);
    driver.process(TOPIC, "1", "1");
    driver.setTime(70);
    driver.process(TOPIC, "1", "1");
    driver.setTime(90);
    driver.process(TOPIC, "1", "1");
    driver.setTime(100);
    driver.process(TOPIC, "1", "1");
    driver.flushState();
    assertEquals(Long.valueOf(2), results.get(new Windowed<>("1", new SessionWindow(10, 30))));
    assertEquals(Long.valueOf(1), results.get(new Windowed<>("2", new SessionWindow(15, 15))));
    assertEquals(Long.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100))));
}