Examples with KGroupedStream org.apache.kafka.streams.kstream.KGroupedStream used on opensource projects

Example 1 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project ksql by confluentinc.

the class HoppingWindowExpressionTest method shouldCreateHoppingWindowAggregate.

@Test
public void shouldCreateHoppingWindowAggregate() {
    final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
    final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
    final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
    final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS);
    final Initializer initializer = () -> 0;
    final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");
    EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream);
    EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
    EasyMock.replay(stream, windowedKStream);
    windowExpression.applyAggregate(stream, initializer, aggregator, store);
    EasyMock.verify(stream, windowedKStream);
}

Example 2 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project ksql by confluentinc.

the class TumblingWindowExpressionTest method shouldCreateTumblingWindowAggregate.

@Test
public void shouldCreateTumblingWindowAggregate() {
    final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
    final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
    final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
    final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS);
    final Initializer initializer = () -> 0;
    final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");
    EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream);
    EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
    EasyMock.replay(stream, windowedKStream);
    windowExpression.applyAggregate(stream, initializer, aggregator, store);
    EasyMock.verify(stream, windowedKStream);
}

Example 3 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project kafka-streams-examples by confluentinc.

the class WordCountInteractiveQueriesExample method createStreams.

static KafkaStreams createStreams(final Properties streamsConfiguration) {
    final Serde<String> stringSerde = Serdes.String();
    StreamsBuilder builder = new StreamsBuilder();
    KStream<String, String> textLines = builder.stream(TEXT_LINES_TOPIC, Consumed.with(Serdes.String(), Serdes.String()));
    final KGroupedStream<String, String> groupedByWord = textLines.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))).groupBy((key, word) -> word, Serialized.with(stringSerde, stringSerde));
    // Create a State Store for with the all time word count
    groupedByWord.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("word-count").withValueSerde(Serdes.Long()));
    // Create a Windowed State Store that contains the word count for every
    // 1 minute
    groupedByWord.windowedBy(TimeWindows.of(60000)).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as("windowed-word-count").withValueSerde(Serdes.Long()));
    return new KafkaStreams(builder.build(), streamsConfiguration);
}

Example 4 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project kafka by apache.

the class CogroupedKStreamImplTest method shouldInsertRepartitionsTopicForCogroupsUsedTwice.

@Test
public void shouldInsertRepartitionsTopicForCogroupsUsedTwice() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties properties = new Properties();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KGroupedStream<String, String> groupedOne = stream1.map((k, v) -> new KeyValue<>(v, k)).groupByKey(Grouped.as("foo"));
    final CogroupedKStream<String, String> one = groupedOne.cogroup(STRING_AGGREGATOR);
    one.aggregate(STRING_INITIALIZER);
    one.aggregate(STRING_INITIALIZER);
    final String topologyDescription = builder.build(properties).describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [one])\n" + "      --> KSTREAM-MAP-0000000001\n" + "    Processor: KSTREAM-MAP-0000000001 (stores: [])\n" + "      --> foo-repartition-filter\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: foo-repartition-filter (stores: [])\n" + "      --> foo-repartition-sink\n" + "      <-- KSTREAM-MAP-0000000001\n" + "    Sink: foo-repartition-sink (topic: foo-repartition)\n" + "      <-- foo-repartition-filter\n\n" + "  Sub-topology: 1\n" + "    Source: foo-repartition-source (topics: [foo-repartition])\n" + "      --> COGROUPKSTREAM-AGGREGATE-0000000006, COGROUPKSTREAM-AGGREGATE-0000000012\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000006 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000002])\n" + "      --> COGROUPKSTREAM-MERGE-0000000007\n" + "      <-- foo-repartition-source\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000012 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000008])\n" + "      --> COGROUPKSTREAM-MERGE-0000000013\n" + "      <-- foo-repartition-source\n" + "    Processor: COGROUPKSTREAM-MERGE-0000000007 (stores: [])\n" + "      --> none\n" + "      <-- COGROUPKSTREAM-AGGREGATE-0000000006\n" + "    Processor: COGROUPKSTREAM-MERGE-0000000013 (stores: [])\n" + "      --> none\n" + "      <-- COGROUPKSTREAM-AGGREGATE-0000000012\n\n"));
}

Example 5 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project kafka by apache.

the class CogroupedKStreamImplTest method shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReusedInSameCogroupsWithOptimization.

@Test
public void shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReusedInSameCogroupsWithOptimization() {
    final Properties properties = new Properties();
    properties.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.OPTIMIZE);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
    final KGroupedStream<String, String> groupedOne = stream1.map((k, v) -> new KeyValue<>(v, k)).groupByKey();
    final KGroupedStream<String, String> groupedTwo = stream2.groupByKey();
    final KTable<String, String> cogroupedTwo = groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
    final KTable<String, String> cogroupedOne = groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
    cogroupedOne.toStream().to(OUTPUT);
    cogroupedTwo.toStream().to("OUTPUT2");
    final String topologyDescription = builder.build(properties).describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [one])\n" + "      --> KSTREAM-MAP-0000000002\n" + "    Processor: KSTREAM-MAP-0000000002 (stores: [])\n" + "      --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter (stores: [])\n" + "      --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink\n" + "      <-- KSTREAM-MAP-0000000002\n" + "    Sink: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink (topic: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition)\n" + "      <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter\n\n" + "  Sub-topology: 1\n" + "    Source: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source (topics: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition])\n" + "      --> COGROUPKSTREAM-AGGREGATE-0000000014, COGROUPKSTREAM-AGGREGATE-0000000007\n" + "    Source: KSTREAM-SOURCE-0000000001 (topics: [two])\n" + "      --> COGROUPKSTREAM-AGGREGATE-0000000015, COGROUPKSTREAM-AGGREGATE-0000000008\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000007 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + "      --> COGROUPKSTREAM-MERGE-0000000009\n" + "      <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000008 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + "      --> COGROUPKSTREAM-MERGE-0000000009\n" + "      <-- KSTREAM-SOURCE-0000000001\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000014 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000010])\n" + "      --> COGROUPKSTREAM-MERGE-0000000016\n" + "      <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source\n" + "    Processor: COGROUPKSTREAM-AGGREGATE-0000000015 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000010])\n" + "      --> COGROUPKSTREAM-MERGE-0000000016\n" + "      <-- KSTREAM-SOURCE-0000000001\n" + "    Processor: COGROUPKSTREAM-MERGE-0000000009 (stores: [])\n" + "      --> KTABLE-TOSTREAM-0000000019\n" + "      <-- COGROUPKSTREAM-AGGREGATE-0000000007, COGROUPKSTREAM-AGGREGATE-0000000008\n" + "    Processor: COGROUPKSTREAM-MERGE-0000000016 (stores: [])\n" + "      --> KTABLE-TOSTREAM-0000000017\n" + "      <-- COGROUPKSTREAM-AGGREGATE-0000000014, COGROUPKSTREAM-AGGREGATE-0000000015\n" + "    Processor: KTABLE-TOSTREAM-0000000017 (stores: [])\n" + "      --> KSTREAM-SINK-0000000018\n" + "      <-- COGROUPKSTREAM-MERGE-0000000016\n" + "    Processor: KTABLE-TOSTREAM-0000000019 (stores: [])\n" + "      --> KSTREAM-SINK-0000000020\n" + "      <-- COGROUPKSTREAM-MERGE-0000000009\n" + "    Sink: KSTREAM-SINK-0000000018 (topic: output)\n" + "      <-- KTABLE-TOSTREAM-0000000017\n" + "    Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT2)\n" + "      <-- KTABLE-TOSTREAM-0000000019\n\n"));
}