Examples with ProcessorParameters org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters used on opensource projects

Example 1 with ProcessorParameters

use of org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters in project kafka by apache.

the class KStreamImplJoin method join.

public <K, V1, V2, VOut> KStream<K, VOut> join(final KStream<K, V1> lhs, final KStream<K, V2> other, final ValueJoinerWithKey<? super K, ? super V1, ? super V2, ? extends VOut> joiner, final JoinWindows windows, final StreamJoined<K, V1, V2> streamJoined) {
    final StreamJoinedInternal<K, V1, V2> streamJoinedInternal = new StreamJoinedInternal<>(streamJoined);
    final NamedInternal renamed = new NamedInternal(streamJoinedInternal.name());
    final String joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join";
    final String joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";
    final String thisWindowStreamProcessorName = renamed.suffixWithOrElseGet("-this-windowed", builder, KStreamImpl.WINDOWED_NAME);
    final String otherWindowStreamProcessorName = renamed.suffixWithOrElseGet("-other-windowed", builder, KStreamImpl.WINDOWED_NAME);
    final String joinThisGeneratedName = rightOuter ? builder.newProcessorName(KStreamImpl.OUTERTHIS_NAME) : builder.newProcessorName(KStreamImpl.JOINTHIS_NAME);
    final String joinOtherGeneratedName = leftOuter ? builder.newProcessorName(KStreamImpl.OUTEROTHER_NAME) : builder.newProcessorName(KStreamImpl.JOINOTHER_NAME);
    final String joinThisName = renamed.suffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
    final String joinOtherName = renamed.suffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);
    final String joinMergeName = renamed.suffixWithOrElseGet("-merge", builder, KStreamImpl.MERGE_NAME);
    final GraphNode thisGraphNode = ((AbstractStream<?, ?>) lhs).graphNode;
    final GraphNode otherGraphNode = ((AbstractStream<?, ?>) other).graphNode;
    final StoreBuilder<WindowStore<K, V1>> thisWindowStore;
    final StoreBuilder<WindowStore<K, V2>> otherWindowStore;
    final String userProvidedBaseStoreName = streamJoinedInternal.storeName();
    final WindowBytesStoreSupplier thisStoreSupplier = streamJoinedInternal.thisStoreSupplier();
    final WindowBytesStoreSupplier otherStoreSupplier = streamJoinedInternal.otherStoreSupplier();
    assertUniqueStoreNames(thisStoreSupplier, otherStoreSupplier);
    if (thisStoreSupplier == null) {
        final String thisJoinStoreName = userProvidedBaseStoreName == null ? joinThisGeneratedName : userProvidedBaseStoreName + joinThisSuffix;
        thisWindowStore = joinWindowStoreBuilder(thisJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde(), streamJoinedInternal.loggingEnabled(), streamJoinedInternal.logConfig());
    } else {
        assertWindowSettings(thisStoreSupplier, windows);
        thisWindowStore = joinWindowStoreBuilderFromSupplier(thisStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
    }
    if (otherStoreSupplier == null) {
        final String otherJoinStoreName = userProvidedBaseStoreName == null ? joinOtherGeneratedName : userProvidedBaseStoreName + joinOtherSuffix;
        otherWindowStore = joinWindowStoreBuilder(otherJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde(), streamJoinedInternal.loggingEnabled(), streamJoinedInternal.logConfig());
    } else {
        assertWindowSettings(otherStoreSupplier, windows);
        otherWindowStore = joinWindowStoreBuilderFromSupplier(otherStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
    }
    final KStreamJoinWindow<K, V1> thisWindowedStream = new KStreamJoinWindow<>(thisWindowStore.name());
    final ProcessorParameters<K, V1, ?, ?> thisWindowStreamProcessorParams = new ProcessorParameters<>(thisWindowedStream, thisWindowStreamProcessorName);
    final ProcessorGraphNode<K, V1> thisWindowedStreamsNode = new ProcessorGraphNode<>(thisWindowStreamProcessorName, thisWindowStreamProcessorParams);
    builder.addGraphNode(thisGraphNode, thisWindowedStreamsNode);
    final KStreamJoinWindow<K, V2> otherWindowedStream = new KStreamJoinWindow<>(otherWindowStore.name());
    final ProcessorParameters<K, V2, ?, ?> otherWindowStreamProcessorParams = new ProcessorParameters<>(otherWindowedStream, otherWindowStreamProcessorName);
    final ProcessorGraphNode<K, V2> otherWindowedStreamsNode = new ProcessorGraphNode<>(otherWindowStreamProcessorName, otherWindowStreamProcessorParams);
    builder.addGraphNode(otherGraphNode, otherWindowedStreamsNode);
    Optional<StoreBuilder<KeyValueStore<TimestampedKeyAndJoinSide<K>, LeftOrRightValue<V1, V2>>>> outerJoinWindowStore = Optional.empty();
    if (leftOuter) {
        outerJoinWindowStore = Optional.of(sharedOuterJoinWindowStoreBuilder(windows, streamJoinedInternal, joinThisGeneratedName));
    }
    // Time-shared between joins to keep track of the maximum stream time
    final TimeTracker sharedTimeTracker = new TimeTracker();
    final JoinWindowsInternal internalWindows = new JoinWindowsInternal(windows);
    final KStreamKStreamJoin<K, V1, V2, VOut> joinThis = new KStreamKStreamJoin<>(true, otherWindowStore.name(), internalWindows, joiner, leftOuter, outerJoinWindowStore.map(StoreBuilder::name), sharedTimeTracker);
    final KStreamKStreamJoin<K, V2, V1, VOut> joinOther = new KStreamKStreamJoin<>(false, thisWindowStore.name(), internalWindows, AbstractStream.reverseJoinerWithKey(joiner), rightOuter, outerJoinWindowStore.map(StoreBuilder::name), sharedTimeTracker);
    final PassThrough<K, VOut> joinMerge = new PassThrough<>();
    final StreamStreamJoinNode.StreamStreamJoinNodeBuilder<K, V1, V2, VOut> joinBuilder = StreamStreamJoinNode.streamStreamJoinNodeBuilder();
    final ProcessorParameters<K, V1, ?, ?> joinThisProcessorParams = new ProcessorParameters<>(joinThis, joinThisName);
    final ProcessorParameters<K, V2, ?, ?> joinOtherProcessorParams = new ProcessorParameters<>(joinOther, joinOtherName);
    final ProcessorParameters<K, VOut, ?, ?> joinMergeProcessorParams = new ProcessorParameters<>(joinMerge, joinMergeName);
    joinBuilder.withJoinMergeProcessorParameters(joinMergeProcessorParams).withJoinThisProcessorParameters(joinThisProcessorParams).withJoinOtherProcessorParameters(joinOtherProcessorParams).withThisWindowStoreBuilder(thisWindowStore).withOtherWindowStoreBuilder(otherWindowStore).withThisWindowedStreamProcessorParameters(thisWindowStreamProcessorParams).withOtherWindowedStreamProcessorParameters(otherWindowStreamProcessorParams).withOuterJoinWindowStoreBuilder(outerJoinWindowStore).withValueJoiner(joiner).withNodeName(joinMergeName);
    if (internalWindows.spuriousResultFixEnabled()) {
        joinBuilder.withSpuriousResultFixEnabled();
    }
    final GraphNode joinGraphNode = joinBuilder.build();
    builder.addGraphNode(Arrays.asList(thisGraphNode, otherGraphNode), joinGraphNode);
    final Set<String> allSourceNodes = new HashSet<>(((KStreamImpl<K, V1>) lhs).subTopologySourceNodes);
    allSourceNodes.addAll(((KStreamImpl<K, V2>) other).subTopologySourceNodes);
    // also for key serde we do not inherit from either since we cannot tell if these two serdes are different
    return new KStreamImpl<>(joinMergeName, streamJoinedInternal.keySerde(), null, allSourceNodes, false, joinGraphNode, builder);
}

Example 2 with ProcessorParameters

use of org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters in project kafka by apache.

the class KStreamImpl method doBranch.

@SuppressWarnings({ "unchecked", "rawtypes" })
private KStream<K, V>[] doBranch(final NamedInternal named, final Predicate<? super K, ? super V>... predicates) {
    Objects.requireNonNull(predicates, "predicates can't be a null array");
    if (predicates.length == 0) {
        throw new IllegalArgumentException("branch() requires at least one predicate");
    }
    for (final Predicate<? super K, ? super V> predicate : predicates) {
        Objects.requireNonNull(predicate, "predicates can't be null");
    }
    final String branchName = named.orElseGenerateWithPrefix(builder, BRANCH_NAME);
    final String[] childNames = new String[predicates.length];
    for (int i = 0; i < predicates.length; i++) {
        childNames[i] = named.suffixWithOrElseGet("-predicate-" + i, builder, BRANCHCHILD_NAME);
    }
    final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamBranch(Arrays.asList(predicates.clone()), Arrays.asList(childNames)), branchName);
    final ProcessorGraphNode<K, V> branchNode = new ProcessorGraphNode<>(branchName, processorParameters);
    builder.addGraphNode(graphNode, branchNode);
    final KStream<K, V>[] branchChildren = (KStream<K, V>[]) Array.newInstance(KStream.class, predicates.length);
    for (int i = 0; i < predicates.length; i++) {
        final ProcessorParameters innerProcessorParameters = new ProcessorParameters<>(new PassThrough<K, V>(), childNames[i]);
        final ProcessorGraphNode<K, V> branchChildNode = new ProcessorGraphNode<>(childNames[i], innerProcessorParameters);
        builder.addGraphNode(branchNode, branchChildNode);
        branchChildren[i] = new KStreamImpl<>(childNames[i], keySerde, valueSerde, subTopologySourceNodes, repartitionRequired, branchChildNode, builder);
    }
    return branchChildren;
}

Example 3 with ProcessorParameters

use of org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters in project kafka by apache.

the class KStreamImpl method toTable.

@Override
public KTable<K, V> toTable(final Named named, final Materialized<K, V, KeyValueStore<Bytes, byte[]>> materialized) {
    Objects.requireNonNull(named, "named can't be null");
    Objects.requireNonNull(materialized, "materialized can't be null");
    final NamedInternal namedInternal = new NamedInternal(named);
    final String name = namedInternal.orElseGenerateWithPrefix(builder, TO_KTABLE_NAME);
    final MaterializedInternal<K, V, KeyValueStore<Bytes, byte[]>> materializedInternal = new MaterializedInternal<>(materialized, builder, TO_KTABLE_NAME);
    final Serde<K> keySerdeOverride = materializedInternal.keySerde() == null ? keySerde : materializedInternal.keySerde();
    final Serde<V> valueSerdeOverride = materializedInternal.valueSerde() == null ? valueSerde : materializedInternal.valueSerde();
    final Set<String> subTopologySourceNodes;
    final GraphNode tableParentNode;
    if (repartitionRequired) {
        final OptimizableRepartitionNodeBuilder<K, V> repartitionNodeBuilder = optimizableRepartitionNodeBuilder();
        final String sourceName = createRepartitionedSource(builder, keySerdeOverride, valueSerdeOverride, name, null, repartitionNodeBuilder);
        tableParentNode = repartitionNodeBuilder.build();
        builder.addGraphNode(graphNode, tableParentNode);
        subTopologySourceNodes = Collections.singleton(sourceName);
    } else {
        tableParentNode = graphNode;
        subTopologySourceNodes = this.subTopologySourceNodes;
    }
    final KTableSource<K, V> tableSource = new KTableSource<>(materializedInternal.storeName(), materializedInternal.queryableStoreName());
    final ProcessorParameters<K, V, ?, ?> processorParameters = new ProcessorParameters<>(tableSource, name);
    final GraphNode tableNode = new StreamToTableNode<>(name, processorParameters, materializedInternal);
    builder.addGraphNode(tableParentNode, tableNode);
    return new KTableImpl<K, V, V>(name, keySerdeOverride, valueSerdeOverride, subTopologySourceNodes, materializedInternal.queryableStoreName(), tableSource, tableNode, builder);
}