Examples with NAryUnionPlanNode org.apache.flink.optimizer.plan.NAryUnionPlanNode used on opensource projects

Example 6 with NAryUnionPlanNode

use of org.apache.flink.optimizer.plan.NAryUnionPlanNode in project flink by apache.

the class JavaApiPostPass method traverse.

protected void traverse(PlanNode node) {
    if (!alreadyDone.add(node)) {
        // already worked on that one
        return;
    }
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
        // descend to the input channel
        SinkPlanNode sn = (SinkPlanNode) node;
        Channel inchannel = sn.getInput();
        traverseChannel(inchannel);
    } else if (node instanceof SourcePlanNode) {
        TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
        ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
    } else if (node instanceof BulkIterationPlanNode) {
        BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
        if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
        }
        // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
        if (iterationNode.getRootOfTerminationCriterion() != null) {
            SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
            traverseChannel(addMapper.getInput());
        }
        BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
        // set the serializer
        iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
        // done, we can now propagate our info down
        traverseChannel(iterationNode.getInput());
        traverse(iterationNode.getRootOfStepFunction());
    } else if (node instanceof WorksetIterationPlanNode) {
        WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
        if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
        }
        if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
        }
        DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
        // set the serializers and comparators for the workset iteration
        iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
        iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
        iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
        // traverse the inputs
        traverseChannel(iterationNode.getInput1());
        traverseChannel(iterationNode.getInput2());
        // traverse the step function
        traverse(iterationNode.getSolutionSetDeltaPlanNode());
        traverse(iterationNode.getNextWorkSetPlanNode());
    } else if (node instanceof SingleInputPlanNode) {
        SingleInputPlanNode sn = (SingleInputPlanNode) node;
        if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
            // Special case for delta iterations
            if (sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
                traverseChannel(sn.getInput());
                return;
            } else {
                throw new RuntimeException("Wrong operator type found in post pass.");
            }
        }
        SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        for (int i = 0; i < sn.getDriverStrategy().getNumRequiredComparators(); i++) {
            sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
        }
        // done, we can now propagate our info down
        traverseChannel(sn.getInput());
        // don't forget the broadcast inputs
        for (Channel c : sn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else if (node instanceof DualInputPlanNode) {
        DualInputPlanNode dn = (DualInputPlanNode) node;
        if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
            throw new RuntimeException("Wrong operator type found in post pass.");
        }
        DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
            dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(), getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
            dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(), getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));
            dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dualInputOperator.getOperatorInfo().getSecondInputType()));
        }
        traverseChannel(dn.getInput1());
        traverseChannel(dn.getInput2());
        // don't forget the broadcast inputs
        for (Channel c : dn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else // catch the sources of the iterative step functions
    if (node instanceof BulkPartialSolutionPlanNode || node instanceof SolutionSetPlanNode || node instanceof WorksetPlanNode) {
    // Do nothing :D
    } else if (node instanceof NAryUnionPlanNode) {
        // Traverse to all child channels
        for (Channel channel : node.getInputs()) {
            traverseChannel(channel);
        }
    } else {
        throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
    }
}

Example 7 with NAryUnionPlanNode

use of org.apache.flink.optimizer.plan.NAryUnionPlanNode in project flink by apache.

the class BinaryUnionReplacer method postVisit.

@Override
public void postVisit(PlanNode visitable) {
    if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
        if (!unionNode.unionsStaticAndDynamicPath()) {
            // both on static path, or both on dynamic path. we can collapse them
            NAryUnionPlanNode newUnionNode;
            List<Channel> inputs = new ArrayList<Channel>();
            collect(in1, inputs);
            collect(in2, inputs);
            newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
            newUnionNode.setParallelism(unionNode.getParallelism());
            for (Channel c : inputs) {
                c.setTarget(newUnionNode);
            }
            for (Channel channel : unionNode.getOutgoingChannels()) {
                channel.swapUnionNodes(newUnionNode);
                newUnionNode.addOutgoingChannel(channel);
            }
        } else {
            // make sure that the first input is the cached (static) and the second input is the dynamic
            if (in1.isOnDynamicPath()) {
                BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
                in1.setTarget(newUnionNode);
                in2.setTarget(newUnionNode);
                for (Channel channel : unionNode.getOutgoingChannels()) {
                    channel.swapUnionNodes(newUnionNode);
                    newUnionNode.addOutgoingChannel(channel);
                }
            }
        }
    }
}

Example 8 with NAryUnionPlanNode

use of org.apache.flink.optimizer.plan.NAryUnionPlanNode in project flink by apache.

the class JobGraphGenerator method postVisit.

/**
	 * This method implements the post-visit during the depth-first traversal. When the post visit happens,
	 * all of the descendants have been processed, so this method connects all of the current node's
	 * predecessors to the current node.
	 * 
	 * @param node
	 *        The node currently processed during the post-visit.
	 * @see org.apache.flink.util.Visitor#postVisit(org.apache.flink.util.Visitable) t
	 */
@Override
public void postVisit(PlanNode node) {
    try {
        // solution sets have no input. the initial solution set input is connected when the iteration node is in its postVisit
        if (node instanceof SourcePlanNode || node instanceof NAryUnionPlanNode || node instanceof SolutionSetPlanNode) {
            return;
        }
        // check if we have an iteration. in that case, translate the step function now
        if (node instanceof IterationPlanNode) {
            // prevent nested iterations
            if (node.isOnDynamicPath()) {
                throw new CompilerException("Nested Iterations are not possible at the moment!");
            }
            // another one), we push the current one onto the stack
            if (this.currentIteration != null) {
                this.iterationStack.add(this.currentIteration);
            }
            this.currentIteration = (IterationPlanNode) node;
            this.currentIteration.acceptForStepFunction(this);
            // pop the current iteration from the stack
            if (this.iterationStack.isEmpty()) {
                this.currentIteration = null;
            } else {
                this.currentIteration = this.iterationStack.remove(this.iterationStack.size() - 1);
            }
            // connect the initial solution set now.
            if (node instanceof WorksetIterationPlanNode) {
                // connect the initial solution set
                WorksetIterationPlanNode wsNode = (WorksetIterationPlanNode) node;
                JobVertex headVertex = this.iterations.get(wsNode).getHeadTask();
                TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
                int inputIndex = headConfig.getDriverStrategy().getNumInputs();
                headConfig.setIterationHeadSolutionSetInputIndex(inputIndex);
                translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
            }
            return;
        }
        final JobVertex targetVertex = this.vertices.get(node);
        // check whether this node has its own task, or is merged with another one
        if (targetVertex == null) {
            // node's task is merged with another task. it is either chained, of a merged head vertex
            // from an iteration
            final TaskInChain chainedTask;
            if ((chainedTask = this.chainedTasks.get(node)) != null) {
                // Chained Task. Sanity check first...
                final Iterator<Channel> inConns = node.getInputs().iterator();
                if (!inConns.hasNext()) {
                    throw new CompilerException("Bug: Found chained task with no input.");
                }
                final Channel inConn = inConns.next();
                if (inConns.hasNext()) {
                    throw new CompilerException("Bug: Found a chained task with more than one input!");
                }
                if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
                    throw new CompilerException("Bug: Found a chained task with an input local strategy.");
                }
                if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
                    throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
                }
                JobVertex container = chainedTask.getContainingVertex();
                if (container == null) {
                    final PlanNode sourceNode = inConn.getSource();
                    container = this.vertices.get(sourceNode);
                    if (container == null) {
                        // predecessor is itself chained
                        container = this.chainedTasks.get(sourceNode).getContainingVertex();
                        if (container == null) {
                            throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
                        }
                    } else {
                        // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
                        new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
                    }
                    chainedTask.setContainingVertex(container);
                }
                // add info about the input serializer type
                chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
                // update name of container task
                String containerTaskName = container.getName();
                if (containerTaskName.startsWith("CHAIN ")) {
                    container.setName(containerTaskName + " -> " + chainedTask.getTaskName());
                } else {
                    container.setName("CHAIN " + containerTaskName + " -> " + chainedTask.getTaskName());
                }
                //update resource of container task
                container.setResources(container.getMinResources().merge(node.getMinResources()), container.getPreferredResources().merge(node.getPreferredResources()));
                this.chainedTasksInSequence.add(chainedTask);
                return;
            } else if (node instanceof BulkPartialSolutionPlanNode || node instanceof WorksetPlanNode) {
                // merged iteration head task. the task that the head is merged with will take care of it
                return;
            } else {
                throw new CompilerException("Bug: Unrecognized merged task vertex.");
            }
        }
        if (this.currentIteration != null) {
            JobVertex head = this.iterations.get(this.currentIteration).getHeadTask();
            // their execution determines the deployment slots of the co-location group
            if (node.isOnDynamicPath()) {
                targetVertex.setStrictlyCoLocatedWith(head);
            }
        }
        // create the config that will contain all the description of the inputs
        final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
        // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
        // enclosing iteration node, because the inputs are the initial inputs to the iteration.
        final Iterator<Channel> inConns;
        if (node instanceof BulkPartialSolutionPlanNode) {
            inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
            // because the partial solution has its own vertex, is has only one (logical) input.
            // note this in the task configuration
            targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
        } else if (node instanceof WorksetPlanNode) {
            WorksetPlanNode wspn = (WorksetPlanNode) node;
            // input that is the initial workset
            inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
            // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
            // be input 0 and the solution set will be input 1
            targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
            targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
        } else {
            inConns = node.getInputs().iterator();
        }
        if (!inConns.hasNext()) {
            throw new CompilerException("Bug: Found a non-source task with no input.");
        }
        int inputIndex = 0;
        while (inConns.hasNext()) {
            Channel input = inConns.next();
            inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
        }
        // broadcast variables
        int broadcastInputIndex = 0;
        for (NamedChannel broadcastInput : node.getBroadcastInputs()) {
            int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true);
            targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
            targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex);
            broadcastInputIndex += broadcastInputIndexDelta;
        }
    } catch (Exception e) {
        throw new CompilerException("An error occurred while translating the optimized plan to a JobGraph: " + e.getMessage(), e);
    }
}

Example 9 with NAryUnionPlanNode

use of org.apache.flink.optimizer.plan.NAryUnionPlanNode in project flink by apache.

the class JobGraphGenerator method createWorksetIterationHead.

private JobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
    // get the bulk iteration that corresponds to this partial solution node
    final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
    // check whether we need an individual vertex for the partial solution, or whether we
    // attach ourselves to the vertex of the parent node. We can combine the head with a node of 
    // the step function, if
    // 1) There is one parent that the partial solution connects to via a forward pattern and no
    //    local strategy
    // 2) parallelism and the number of subtasks per instance does not change
    // 3) That successor is not a union
    // 4) That successor is not itself the last node of the step function
    // 5) There is no local strategy on the edge for the initial workset, as
    //    this translates to a local strategy that would only be executed in the first superstep
    final boolean merge;
    if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
        final Channel c = wspn.getOutgoingChannels().get(0);
        final PlanNode successor = c.getTarget();
        merge = c.getShipStrategy() == ShipStrategyType.FORWARD && c.getLocalStrategy() == LocalStrategy.NONE && c.getTempMode() == TempMode.NONE && successor.getParallelism() == wspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getNextWorkSetPlanNode() && iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
        merge = false;
    }
    // create or adopt the head vertex
    final JobVertex toReturn;
    final JobVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
        final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
        headVertex = this.vertices.get(successor);
        if (headVertex == null) {
            throw new CompilerException("Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
        }
        // reset the vertex type to iteration head
        headVertex.setInvokableClass(IterationHeadTask.class);
        headConfig = new TaskConfig(headVertex.getConfiguration());
        toReturn = null;
    } else {
        // instantiate the head vertex and give it a no-op driver as the driver strategy.
        // everything else happens in the post visit, after the input (the initial partial solution)
        // is connected.
        headVertex = new JobVertex("IterationHead(" + iteration.getNodeName() + ")");
        headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources());
        headVertex.setInvokableClass(IterationHeadTask.class);
        headConfig = new TaskConfig(headVertex.getConfiguration());
        headConfig.setDriver(NoOpDriver.class);
        toReturn = headVertex;
    }
    headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
    // create the iteration descriptor and the iteration to it
    IterationDescriptor descr = this.iterations.get(iteration);
    if (descr == null) {
        throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
    }
    descr.setHeadTask(headVertex, headConfig);
    return toReturn;
}

Example 10 with NAryUnionPlanNode

use of org.apache.flink.optimizer.plan.NAryUnionPlanNode in project flink by apache.

the class JobGraphGenerator method createSingleInputVertex.

// ------------------------------------------------------------------------
// Methods for creating individual vertices
// ------------------------------------------------------------------------
private JobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
    final String taskName = node.getNodeName();
    final DriverStrategy ds = node.getDriverStrategy();
    // check, whether chaining is possible
    boolean chaining;
    {
        Channel inConn = node.getInput();
        PlanNode pred = inConn.getSource();
        chaining = ds.getPushChainDriverClass() != null && // first op after union is stand-alone, because union is merged
        !(pred instanceof NAryUnionPlanNode) && // partial solution merges anyways
        !(pred instanceof BulkPartialSolutionPlanNode) && // workset merges anyways
        !(pred instanceof WorksetPlanNode) && // cannot chain with iteration heads currently
        !(pred instanceof IterationPlanNode) && inConn.getShipStrategy() == ShipStrategyType.FORWARD && inConn.getLocalStrategy() == LocalStrategy.NONE && pred.getOutgoingChannels().size() == 1 && node.getParallelism() == pred.getParallelism() && node.getBroadcastInputs().isEmpty();
        // in a tail 
        if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode && node.getOutgoingChannels().size() > 0) {
            WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
            if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
                chaining = false;
            }
        }
        // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
        if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode) {
            BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
            if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred) {
                chaining = false;
            } else if (node.getOutgoingChannels().size() > 0 && (wspn.getRootOfStepFunction() == pred || wspn.getRootOfTerminationCriterion() == pred)) {
                chaining = false;
            }
        }
    }
    final JobVertex vertex;
    final TaskConfig config;
    if (chaining) {
        vertex = null;
        config = new TaskConfig(new Configuration());
        this.chainedTasks.put(node, new TaskInChain(node, ds.getPushChainDriverClass(), config, taskName));
    } else {
        // create task vertex
        vertex = new JobVertex(taskName);
        vertex.setResources(node.getMinResources(), node.getPreferredResources());
        vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class);
        config = new TaskConfig(vertex.getConfiguration());
        config.setDriver(ds.getDriverClass());
    }
    // set user code
    config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
    config.setStubParameters(node.getProgramOperator().getParameters());
    // set the driver strategy
    config.setDriverStrategy(ds);
    for (int i = 0; i < ds.getNumRequiredComparators(); i++) {
        config.setDriverComparator(node.getComparator(i), i);
    }
    // assign memory, file-handles, etc.
    assignDriverResources(node, config);
    return vertex;
}