Examples with OptimizableRepartitionNode org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode used on opensource projects

Example 1 with OptimizableRepartitionNode

use of org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode in project kafka by apache.

the class InternalStreamsBuilder method maybeUpdateKeyChangingRepartitionNodeMap.

private void maybeUpdateKeyChangingRepartitionNodeMap() {
    final Map<GraphNode, Set<GraphNode>> mergeNodesToKeyChangers = new HashMap<>();
    final Set<GraphNode> mergeNodeKeyChangingParentsToRemove = new HashSet<>();
    for (final GraphNode mergeNode : mergeNodes) {
        mergeNodesToKeyChangers.put(mergeNode, new LinkedHashSet<>());
        final Set<Map.Entry<GraphNode, LinkedHashSet<OptimizableRepartitionNode<?, ?>>>> entrySet = keyChangingOperationsToOptimizableRepartitionNodes.entrySet();
        for (final Map.Entry<GraphNode, LinkedHashSet<OptimizableRepartitionNode<?, ?>>> entry : entrySet) {
            if (mergeNodeHasRepartitionChildren(mergeNode, entry.getValue())) {
                final GraphNode maybeParentKey = findParentNodeMatching(mergeNode, node -> node.parentNodes().contains(entry.getKey()));
                if (maybeParentKey != null) {
                    mergeNodesToKeyChangers.get(mergeNode).add(entry.getKey());
                }
            }
        }
    }
    for (final Map.Entry<GraphNode, Set<GraphNode>> entry : mergeNodesToKeyChangers.entrySet()) {
        final GraphNode mergeKey = entry.getKey();
        final Collection<GraphNode> keyChangingParents = entry.getValue();
        final LinkedHashSet<OptimizableRepartitionNode<?, ?>> repartitionNodes = new LinkedHashSet<>();
        for (final GraphNode keyChangingParent : keyChangingParents) {
            repartitionNodes.addAll(keyChangingOperationsToOptimizableRepartitionNodes.get(keyChangingParent));
            mergeNodeKeyChangingParentsToRemove.add(keyChangingParent);
        }
        keyChangingOperationsToOptimizableRepartitionNodes.put(mergeKey, repartitionNodes);
    }
    for (final GraphNode mergeNodeKeyChangingParent : mergeNodeKeyChangingParentsToRemove) {
        keyChangingOperationsToOptimizableRepartitionNodes.remove(mergeNodeKeyChangingParent);
    }
}

Example 2 with OptimizableRepartitionNode

use of org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode in project kafka by apache.

the class InternalStreamsBuilder method maybeOptimizeRepartitionOperations.

private void maybeOptimizeRepartitionOperations() {
    maybeUpdateKeyChangingRepartitionNodeMap();
    final Iterator<Entry<GraphNode, LinkedHashSet<OptimizableRepartitionNode<?, ?>>>> entryIterator = keyChangingOperationsToOptimizableRepartitionNodes.entrySet().iterator();
    while (entryIterator.hasNext()) {
        final Map.Entry<GraphNode, LinkedHashSet<OptimizableRepartitionNode<?, ?>>> entry = entryIterator.next();
        final GraphNode keyChangingNode = entry.getKey();
        if (entry.getValue().isEmpty()) {
            continue;
        }
        final GroupedInternal<?, ?> groupedInternal = new GroupedInternal<>(getRepartitionSerdes(entry.getValue()));
        final String repartitionTopicName = getFirstRepartitionTopicName(entry.getValue());
        // passing in the name of the first repartition topic, re-used to create the optimized repartition topic
        final GraphNode optimizedSingleRepartition = createRepartitionNode(repartitionTopicName, groupedInternal.keySerde(), groupedInternal.valueSerde());
        // re-use parent buildPriority to make sure the single repartition graph node is evaluated before downstream nodes
        optimizedSingleRepartition.setBuildPriority(keyChangingNode.buildPriority());
        for (final OptimizableRepartitionNode<?, ?> repartitionNodeToBeReplaced : entry.getValue()) {
            final GraphNode keyChangingNodeChild = findParentNodeMatching(repartitionNodeToBeReplaced, gn -> gn.parentNodes().contains(keyChangingNode));
            if (keyChangingNodeChild == null) {
                throw new StreamsException(String.format("Found a null keyChangingChild node for %s", repartitionNodeToBeReplaced));
            }
            LOG.debug("Found the child node of the key changer {} from the repartition {}.", keyChangingNodeChild, repartitionNodeToBeReplaced);
            // need to add children of key-changing node as children of optimized repartition
            // in order to process records from re-partitioning
            optimizedSingleRepartition.addChild(keyChangingNodeChild);
            LOG.debug("Removing {} from {}  children {}", keyChangingNodeChild, keyChangingNode, keyChangingNode.children());
            // now remove children from key-changing node
            keyChangingNode.removeChild(keyChangingNodeChild);
            // now need to get children of repartition node so we can remove repartition node
            final Collection<GraphNode> repartitionNodeToBeReplacedChildren = repartitionNodeToBeReplaced.children();
            final Collection<GraphNode> parentsOfRepartitionNodeToBeReplaced = repartitionNodeToBeReplaced.parentNodes();
            for (final GraphNode repartitionNodeToBeReplacedChild : repartitionNodeToBeReplacedChildren) {
                for (final GraphNode parentNode : parentsOfRepartitionNodeToBeReplaced) {
                    parentNode.addChild(repartitionNodeToBeReplacedChild);
                }
            }
            for (final GraphNode parentNode : parentsOfRepartitionNodeToBeReplaced) {
                parentNode.removeChild(repartitionNodeToBeReplaced);
            }
            repartitionNodeToBeReplaced.clearChildren();
            // if replaced repartition node is part of any copartition group,
            // we need to update it with the new node name so that co-partitioning won't break.
            internalTopologyBuilder.maybeUpdateCopartitionSourceGroups(repartitionNodeToBeReplaced.nodeName(), optimizedSingleRepartition.nodeName());
            LOG.debug("Updated node {} children {}", optimizedSingleRepartition, optimizedSingleRepartition.children());
        }
        keyChangingNode.addChild(optimizedSingleRepartition);
        entryIterator.remove();
    }
}