Examples with KGroupedStream org.apache.kafka.streams.kstream.KGroupedStream used on opensource projects

Example 16 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project ksql by confluentinc.

the class SchemaKStream method groupBy.

public SchemaKGroupedStream groupBy(final Serde<String> keySerde, final Serde<GenericRow> valSerde, final List<Expression> groupByExpressions) {
    boolean rekey = rekeyRequired(groupByExpressions);
    if (!rekey) {
        KGroupedStream kgroupedStream = kstream.groupByKey(Serialized.with(keySerde, valSerde));
        return new SchemaKGroupedStream(schema, kgroupedStream, keyField, Collections.singletonList(this), functionRegistry, schemaRegistryClient);
    }
    // Collect the column indexes, and build the new key as <column1>+<column2>+...
    StringBuilder aggregateKeyName = new StringBuilder();
    List<Integer> newKeyIndexes = new ArrayList<>();
    boolean addSeparator = false;
    for (Expression groupByExpr : groupByExpressions) {
        if (addSeparator) {
            aggregateKeyName.append("|+|");
        } else {
            addSeparator = true;
        }
        aggregateKeyName.append(groupByExpr.toString());
        newKeyIndexes.add(SchemaUtil.getIndexInSchema(groupByExpr.toString(), getSchema()));
    }
    KGroupedStream kgroupedStream = kstream.filter((key, value) -> value != null).groupBy((key, value) -> {
        StringBuilder newKey = new StringBuilder();
        boolean addSeparator1 = false;
        for (int index : newKeyIndexes) {
            if (addSeparator1) {
                newKey.append("|+|");
            } else {
                addSeparator1 = true;
            }
            newKey.append(String.valueOf(value.getColumns().get(index)));
        }
        return newKey.toString();
    }, Serialized.with(keySerde, valSerde));
    // TODO: if the key is a prefix of the grouping columns then we can
    // use the repartition reflection hack to tell streams not to
    // repartition.
    Field newKeyField = new Field(aggregateKeyName.toString(), -1, Schema.STRING_SCHEMA);
    return new SchemaKGroupedStream(schema, kgroupedStream, newKeyField, Collections.singletonList(this), functionRegistry, schemaRegistryClient);
}

Example 17 with KGroupedStream

use of org.apache.kafka.streams.kstream.KGroupedStream in project kafka by apache.

the class CogroupedKStreamImplTest method shouldInsertRepartitionsTopicForUpstreamKeyModification.

@Test
public void shouldInsertRepartitionsTopicForUpstreamKeyModification() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KStream<String, String> test2 = builder.stream("two", stringConsumed);
    final KGroupedStream<String, String> groupedOne = stream1.map((k, v) -> new KeyValue<>(v, k)).groupByKey();
    final KGroupedStream<String, String> groupedTwo = test2.groupByKey();
    final KTable<String, String> customers = groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER, Named.as("test"), Materialized.as("store"));
    customers.toStream().to(OUTPUT);
    final String topologyDescription = builder.build().describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [one])\n" + "      --> KSTREAM-MAP-0000000002\n" + "    Processor: KSTREAM-MAP-0000000002 (stores: [])\n" + "      --> store-repartition-filter\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: store-repartition-filter (stores: [])\n" + "      --> store-repartition-sink\n" + "      <-- KSTREAM-MAP-0000000002\n" + "    Sink: store-repartition-sink (topic: store-repartition)\n" + "      <-- store-repartition-filter\n\n" + "  Sub-topology: 1\n" + "    Source: KSTREAM-SOURCE-0000000001 (topics: [two])\n" + "      --> test-cogroup-agg-1\n" + "    Source: store-repartition-source (topics: [store-repartition])\n" + "      --> test-cogroup-agg-0\n" + "    Processor: test-cogroup-agg-0 (stores: [store])\n" + "      --> test-cogroup-merge\n" + "      <-- store-repartition-source\n" + "    Processor: test-cogroup-agg-1 (stores: [store])\n" + "      --> test-cogroup-merge\n" + "      <-- KSTREAM-SOURCE-0000000001\n" + "    Processor: test-cogroup-merge (stores: [])\n" + "      --> KTABLE-TOSTREAM-0000000009\n" + "      <-- test-cogroup-agg-0, test-cogroup-agg-1\n" + "    Processor: KTABLE-TOSTREAM-0000000009 (stores: [])\n" + "      --> KSTREAM-SINK-0000000010\n" + "      <-- test-cogroup-merge\n" + "    Sink: KSTREAM-SINK-0000000010 (topic: output)\n" + "      <-- KTABLE-TOSTREAM-0000000009\n\n"));
}