Examples with JoinWindows org.apache.kafka.streams.kstream.JoinWindows used on opensource projects

Example 11 with JoinWindows

use of org.apache.kafka.streams.kstream.JoinWindows in project kafka by apache.

the class TopologyTest method streamStreamOuterJoinTopologyWithCustomStoresSuppliers.

@Test
public void streamStreamOuterJoinTopologyWithCustomStoresSuppliers() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, String> stream1;
    final KStream<Integer, String> stream2;
    stream1 = builder.stream("input-topic1");
    stream2 = builder.stream("input-topic2");
    final JoinWindows joinWindows = JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100));
    final WindowBytesStoreSupplier thisStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    final WindowBytesStoreSupplier otherStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store-other", Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()), Duration.ofMillis(joinWindows.size()), true);
    stream1.outerJoin(stream2, MockValueJoiner.TOSTRING_JOINER, joinWindows, StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()).withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier));
    final TopologyDescription describe = builder.build().describe();
    assertEquals("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input-topic1])\n" + "      --> KSTREAM-WINDOWED-0000000002\n" + "    Source: KSTREAM-SOURCE-0000000001 (topics: [input-topic2])\n" + "      --> KSTREAM-WINDOWED-0000000003\n" + "    Processor: KSTREAM-WINDOWED-0000000002 (stores: [in-memory-join-store])\n" + "      --> KSTREAM-OUTERTHIS-0000000004\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: KSTREAM-WINDOWED-0000000003 (stores: [in-memory-join-store-other])\n" + "      --> KSTREAM-OUTEROTHER-0000000005\n" + "      <-- KSTREAM-SOURCE-0000000001\n" + "    Processor: KSTREAM-OUTEROTHER-0000000005 (stores: [in-memory-join-store-outer-shared-join-store, in-memory-join-store])\n" + "      --> KSTREAM-MERGE-0000000006\n" + "      <-- KSTREAM-WINDOWED-0000000003\n" + "    Processor: KSTREAM-OUTERTHIS-0000000004 (stores: [in-memory-join-store-other, in-memory-join-store-outer-shared-join-store])\n" + "      --> KSTREAM-MERGE-0000000006\n" + "      <-- KSTREAM-WINDOWED-0000000002\n" + "    Processor: KSTREAM-MERGE-0000000006 (stores: [])\n" + "      --> none\n" + "      <-- KSTREAM-OUTERTHIS-0000000004, KSTREAM-OUTEROTHER-0000000005\n\n", describe.toString());
}

Example 12 with JoinWindows

use of org.apache.kafka.streams.kstream.JoinWindows in project ksql by confluentinc.

the class SchemaKStreamTest method shouldBuildStepForStreamStreamJoin.

@Test
// can be fixed after GRACE clause is made mandatory
@SuppressWarnings({ "rawtypes", "deprecation" })
public void shouldBuildStepForStreamStreamJoin() {
    // Given:
    final SchemaKStream initialSchemaKStream = buildSchemaKStreamForJoin(ksqlStream);
    final List<Pair<JoinType, StreamStreamJoin>> cases = ImmutableList.of(Pair.of(JoinType.OUTER, initialSchemaKStream::outerJoin), Pair.of(JoinType.LEFT, initialSchemaKStream::leftJoin), Pair.of(JoinType.INNER, initialSchemaKStream::innerJoin));
    final JoinWindows joinWindows = JoinWindows.of(Duration.ofSeconds(1));
    final WindowTimeClause grace = new WindowTimeClause(5, TimeUnit.SECONDS);
    final WithinExpression withinExpression = new WithinExpression(1, TimeUnit.SECONDS, grace);
    for (final Pair<JoinType, StreamStreamJoin> testcase : cases) {
        final SchemaKStream joinedKStream = testcase.right.join(schemaKStream, KEY, withinExpression, valueFormat.getFormatInfo(), valueFormat.getFormatInfo(), childContextStacker);
        // Then:
        assertThat(joinedKStream.getSourceStep(), equalTo(ExecutionStepFactory.streamStreamJoin(childContextStacker, testcase.left, KEY, InternalFormats.of(keyFormat, valueFormat.getFormatInfo()), InternalFormats.of(keyFormat, valueFormat.getFormatInfo()), initialSchemaKStream.getSourceStep(), schemaKStream.getSourceStep(), joinWindows, Optional.of(grace))));
    }
}

Example 13 with JoinWindows

use of org.apache.kafka.streams.kstream.JoinWindows in project ksql by confluentinc.

the class StreamStreamJoinBuilder method build.

// deprecation can be fixed after GRACE clause is mandatory
// (cf. `WithinExpression`)
@SuppressWarnings("deprecation")
public static <K> KStreamHolder<K> build(final KStreamHolder<K> left, final KStreamHolder<K> right, final StreamStreamJoin<K> join, final RuntimeBuildContext buildContext, final StreamJoinedFactory streamJoinedFactory) {
    final Formats leftFormats = join.getLeftInternalFormats();
    final QueryContext queryContext = join.getProperties().getQueryContext();
    final QueryContext.Stacker stacker = QueryContext.Stacker.of(queryContext);
    final LogicalSchema leftSchema = left.getSchema();
    final PhysicalSchema leftPhysicalSchema = PhysicalSchema.from(leftSchema, leftFormats.getKeyFeatures(), leftFormats.getValueFeatures());
    final Serde<GenericRow> leftSerde = buildContext.buildValueSerde(leftFormats.getValueFormat(), leftPhysicalSchema, stacker.push(LEFT_SERDE_CTX).getQueryContext());
    final Formats rightFormats = join.getRightInternalFormats();
    final LogicalSchema rightSchema = right.getSchema();
    final PhysicalSchema rightPhysicalSchema = PhysicalSchema.from(rightSchema, rightFormats.getKeyFeatures(), rightFormats.getValueFeatures());
    final Serde<GenericRow> rightSerde = buildContext.buildValueSerde(rightFormats.getValueFormat(), rightPhysicalSchema, stacker.push(RIGHT_SERDE_CTX).getQueryContext());
    final Serde<K> keySerde = left.getExecutionKeyFactory().buildKeySerde(leftFormats.getKeyFormat(), leftPhysicalSchema, queryContext);
    final StreamJoined<K, GenericRow, GenericRow> joined = streamJoinedFactory.create(keySerde, leftSerde, rightSerde, StreamsUtil.buildOpName(queryContext), StreamsUtil.buildOpName(queryContext));
    final JoinParams joinParams = JoinParamsFactory.create(join.getKeyColName(), leftSchema, rightSchema);
    JoinWindows joinWindows;
    // which enables the "spurious" results bugfix with left/outer joins (see KAFKA-10847).
    if (join.getGraceMillis().isPresent()) {
        joinWindows = JoinWindows.ofTimeDifferenceAndGrace(join.getBeforeMillis(), join.getGraceMillis().get());
    } else {
        joinWindows = JoinWindows.of(join.getBeforeMillis());
    }
    joinWindows = joinWindows.after(join.getAfterMillis());
    final KStream<K, GenericRow> result;
    switch(join.getJoinType()) {
        case LEFT:
            result = left.getStream().leftJoin(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        case OUTER:
            result = left.getStream().outerJoin(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        case INNER:
            result = left.getStream().join(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        default:
            throw new IllegalStateException("invalid join type");
    }
    return left.withStream(result, joinParams.getSchema());
}