Examples with StoreBuilder org.apache.kafka.streams.state.StoreBuilder used on opensource projects

Example 11 with StoreBuilder

use of org.apache.kafka.streams.state.StoreBuilder in project kafka by apache.

the class KStreamImplJoin method join.

public <K, V1, V2, VOut> KStream<K, VOut> join(final KStream<K, V1> lhs, final KStream<K, V2> other, final ValueJoinerWithKey<? super K, ? super V1, ? super V2, ? extends VOut> joiner, final JoinWindows windows, final StreamJoined<K, V1, V2> streamJoined) {
    final StreamJoinedInternal<K, V1, V2> streamJoinedInternal = new StreamJoinedInternal<>(streamJoined);
    final NamedInternal renamed = new NamedInternal(streamJoinedInternal.name());
    final String joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join";
    final String joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";
    final String thisWindowStreamProcessorName = renamed.suffixWithOrElseGet("-this-windowed", builder, KStreamImpl.WINDOWED_NAME);
    final String otherWindowStreamProcessorName = renamed.suffixWithOrElseGet("-other-windowed", builder, KStreamImpl.WINDOWED_NAME);
    final String joinThisGeneratedName = rightOuter ? builder.newProcessorName(KStreamImpl.OUTERTHIS_NAME) : builder.newProcessorName(KStreamImpl.JOINTHIS_NAME);
    final String joinOtherGeneratedName = leftOuter ? builder.newProcessorName(KStreamImpl.OUTEROTHER_NAME) : builder.newProcessorName(KStreamImpl.JOINOTHER_NAME);
    final String joinThisName = renamed.suffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
    final String joinOtherName = renamed.suffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);
    final String joinMergeName = renamed.suffixWithOrElseGet("-merge", builder, KStreamImpl.MERGE_NAME);
    final GraphNode thisGraphNode = ((AbstractStream<?, ?>) lhs).graphNode;
    final GraphNode otherGraphNode = ((AbstractStream<?, ?>) other).graphNode;
    final StoreBuilder<WindowStore<K, V1>> thisWindowStore;
    final StoreBuilder<WindowStore<K, V2>> otherWindowStore;
    final String userProvidedBaseStoreName = streamJoinedInternal.storeName();
    final WindowBytesStoreSupplier thisStoreSupplier = streamJoinedInternal.thisStoreSupplier();
    final WindowBytesStoreSupplier otherStoreSupplier = streamJoinedInternal.otherStoreSupplier();
    assertUniqueStoreNames(thisStoreSupplier, otherStoreSupplier);
    if (thisStoreSupplier == null) {
        final String thisJoinStoreName = userProvidedBaseStoreName == null ? joinThisGeneratedName : userProvidedBaseStoreName + joinThisSuffix;
        thisWindowStore = joinWindowStoreBuilder(thisJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde(), streamJoinedInternal.loggingEnabled(), streamJoinedInternal.logConfig());
    } else {
        assertWindowSettings(thisStoreSupplier, windows);
        thisWindowStore = joinWindowStoreBuilderFromSupplier(thisStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
    }
    if (otherStoreSupplier == null) {
        final String otherJoinStoreName = userProvidedBaseStoreName == null ? joinOtherGeneratedName : userProvidedBaseStoreName + joinOtherSuffix;
        otherWindowStore = joinWindowStoreBuilder(otherJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde(), streamJoinedInternal.loggingEnabled(), streamJoinedInternal.logConfig());
    } else {
        assertWindowSettings(otherStoreSupplier, windows);
        otherWindowStore = joinWindowStoreBuilderFromSupplier(otherStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
    }
    final KStreamJoinWindow<K, V1> thisWindowedStream = new KStreamJoinWindow<>(thisWindowStore.name());
    final ProcessorParameters<K, V1, ?, ?> thisWindowStreamProcessorParams = new ProcessorParameters<>(thisWindowedStream, thisWindowStreamProcessorName);
    final ProcessorGraphNode<K, V1> thisWindowedStreamsNode = new ProcessorGraphNode<>(thisWindowStreamProcessorName, thisWindowStreamProcessorParams);
    builder.addGraphNode(thisGraphNode, thisWindowedStreamsNode);
    final KStreamJoinWindow<K, V2> otherWindowedStream = new KStreamJoinWindow<>(otherWindowStore.name());
    final ProcessorParameters<K, V2, ?, ?> otherWindowStreamProcessorParams = new ProcessorParameters<>(otherWindowedStream, otherWindowStreamProcessorName);
    final ProcessorGraphNode<K, V2> otherWindowedStreamsNode = new ProcessorGraphNode<>(otherWindowStreamProcessorName, otherWindowStreamProcessorParams);
    builder.addGraphNode(otherGraphNode, otherWindowedStreamsNode);
    Optional<StoreBuilder<KeyValueStore<TimestampedKeyAndJoinSide<K>, LeftOrRightValue<V1, V2>>>> outerJoinWindowStore = Optional.empty();
    if (leftOuter) {
        outerJoinWindowStore = Optional.of(sharedOuterJoinWindowStoreBuilder(windows, streamJoinedInternal, joinThisGeneratedName));
    }
    // Time-shared between joins to keep track of the maximum stream time
    final TimeTracker sharedTimeTracker = new TimeTracker();
    final JoinWindowsInternal internalWindows = new JoinWindowsInternal(windows);
    final KStreamKStreamJoin<K, V1, V2, VOut> joinThis = new KStreamKStreamJoin<>(true, otherWindowStore.name(), internalWindows, joiner, leftOuter, outerJoinWindowStore.map(StoreBuilder::name), sharedTimeTracker);
    final KStreamKStreamJoin<K, V2, V1, VOut> joinOther = new KStreamKStreamJoin<>(false, thisWindowStore.name(), internalWindows, AbstractStream.reverseJoinerWithKey(joiner), rightOuter, outerJoinWindowStore.map(StoreBuilder::name), sharedTimeTracker);
    final PassThrough<K, VOut> joinMerge = new PassThrough<>();
    final StreamStreamJoinNode.StreamStreamJoinNodeBuilder<K, V1, V2, VOut> joinBuilder = StreamStreamJoinNode.streamStreamJoinNodeBuilder();
    final ProcessorParameters<K, V1, ?, ?> joinThisProcessorParams = new ProcessorParameters<>(joinThis, joinThisName);
    final ProcessorParameters<K, V2, ?, ?> joinOtherProcessorParams = new ProcessorParameters<>(joinOther, joinOtherName);
    final ProcessorParameters<K, VOut, ?, ?> joinMergeProcessorParams = new ProcessorParameters<>(joinMerge, joinMergeName);
    joinBuilder.withJoinMergeProcessorParameters(joinMergeProcessorParams).withJoinThisProcessorParameters(joinThisProcessorParams).withJoinOtherProcessorParameters(joinOtherProcessorParams).withThisWindowStoreBuilder(thisWindowStore).withOtherWindowStoreBuilder(otherWindowStore).withThisWindowedStreamProcessorParameters(thisWindowStreamProcessorParams).withOtherWindowedStreamProcessorParameters(otherWindowStreamProcessorParams).withOuterJoinWindowStoreBuilder(outerJoinWindowStore).withValueJoiner(joiner).withNodeName(joinMergeName);
    if (internalWindows.spuriousResultFixEnabled()) {
        joinBuilder.withSpuriousResultFixEnabled();
    }
    final GraphNode joinGraphNode = joinBuilder.build();
    builder.addGraphNode(Arrays.asList(thisGraphNode, otherGraphNode), joinGraphNode);
    final Set<String> allSourceNodes = new HashSet<>(((KStreamImpl<K, V1>) lhs).subTopologySourceNodes);
    allSourceNodes.addAll(((KStreamImpl<K, V2>) other).subTopologySourceNodes);
    // also for key serde we do not inherit from either since we cannot tell if these two serdes are different
    return new KStreamImpl<>(joinMergeName, streamJoinedInternal.keySerde(), null, allSourceNodes, false, joinGraphNode, builder);
}

Example 12 with StoreBuilder

use of org.apache.kafka.streams.state.StoreBuilder in project kafka by apache.

the class Topology method addProcessor.

/**
 * Add a new processor node that receives and processes records output by one or more parent source or processor
 * node.
 * Any new record output by this processor will be forwarded to its child processor or sink nodes.
 * If {@code supplier} provides stores via {@link ConnectedStoreProvider#stores()}, the provided {@link StoreBuilder}s
 * will be added to the topology and connected to this processor automatically.
 *
 * @param name the unique name of the processor node
 * @param supplier the supplier used to obtain this node's {@link Processor} instance
 * @param parentNames the name of one or more source or processor nodes whose output records this processor should receive
 * and process
 * @return itself
 * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
 */
public synchronized <KIn, VIn, KOut, VOut> Topology addProcessor(final String name, final ProcessorSupplier<KIn, VIn, KOut, VOut> supplier, final String... parentNames) {
    internalTopologyBuilder.addProcessor(name, supplier, parentNames);
    final Set<StoreBuilder<?>> stores = supplier.stores();
    if (stores != null) {
        for (final StoreBuilder storeBuilder : stores) {
            internalTopologyBuilder.addStateStore(storeBuilder, name);
        }
    }
    return this;
}

Example 13 with StoreBuilder

use of org.apache.kafka.streams.state.StoreBuilder in project kafka by apache.

the class TopologyTest method shouldNotAllowToAddStoreWithSameNameAndDifferentInstance.

@Test
public void shouldNotAllowToAddStoreWithSameNameAndDifferentInstance() {
    mockStoreBuilder();
    EasyMock.replay(storeBuilder);
    topology.addStateStore(storeBuilder);
    final StoreBuilder otherStoreBuilder = EasyMock.createNiceMock(StoreBuilder.class);
    EasyMock.expect(otherStoreBuilder.name()).andReturn("store").anyTimes();
    EasyMock.expect(otherStoreBuilder.logConfig()).andReturn(Collections.emptyMap());
    EasyMock.expect(otherStoreBuilder.loggingEnabled()).andReturn(false);
    EasyMock.replay(otherStoreBuilder);
    try {
        topology.addStateStore(otherStoreBuilder);
        fail("Should have thrown TopologyException for same store name with different StoreBuilder");
    } catch (final TopologyException expected) {
    }
}