Examples with Channel org.apache.flink.optimizer.plan.Channel used on opensource projects

Example 21 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class BinaryUnionReplacer method postVisit.

@Override
public void postVisit(PlanNode visitable) {
    if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
        if (!unionNode.unionsStaticAndDynamicPath()) {
            // both on static path, or both on dynamic path. we can collapse them
            NAryUnionPlanNode newUnionNode;
            List<Channel> inputs = new ArrayList<Channel>();
            collect(in1, inputs);
            collect(in2, inputs);
            newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
            newUnionNode.setParallelism(unionNode.getParallelism());
            for (Channel c : inputs) {
                c.setTarget(newUnionNode);
            }
            for (Channel channel : unionNode.getOutgoingChannels()) {
                channel.swapUnionNodes(newUnionNode);
                newUnionNode.addOutgoingChannel(channel);
            }
        } else {
            // make sure that the first input is the cached (static) and the second input is the dynamic
            if (in1.isOnDynamicPath()) {
                BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
                in1.setTarget(newUnionNode);
                in2.setTarget(newUnionNode);
                for (Channel channel : unionNode.getOutgoingChannels()) {
                    channel.swapUnionNodes(newUnionNode);
                    newUnionNode.addOutgoingChannel(channel);
                }
            }
        }
    }
}

Example 22 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class PlanFinalizer method createFinalPlan.

public OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
    this.memoryConsumerWeights = 0;
    // traverse the graph
    for (SinkPlanNode node : sinks) {
        node.accept(this);
    }
    // assign the memory to each node
    if (this.memoryConsumerWeights > 0) {
        for (PlanNode node : this.allNodes) {
            // assign memory to the driver strategy of the node
            final int consumerWeight = node.getMemoryConsumerWeight();
            if (consumerWeight > 0) {
                final double relativeMem = (double) consumerWeight / this.memoryConsumerWeights;
                node.setRelativeMemoryPerSubtask(relativeMem);
                if (Optimizer.LOG.isDebugEnabled()) {
                    Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " + node.getProgramOperator().getName() + ".");
                }
            }
            // assign memory to the local and global strategies of the channels
            for (Channel c : node.getInputs()) {
                if (c.getLocalStrategy().dams()) {
                    final double relativeMem = 1.0 / this.memoryConsumerWeights;
                    c.setRelativeMemoryLocalStrategy(relativeMem);
                    if (Optimizer.LOG.isDebugEnabled()) {
                        Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " + "instance of " + c + ".");
                    }
                }
                if (c.getTempMode() != TempMode.NONE) {
                    final double relativeMem = 1.0 / this.memoryConsumerWeights;
                    c.setRelativeTempMemory(relativeMem);
                    if (Optimizer.LOG.isDebugEnabled()) {
                        Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " + "table for " + c + ".");
                    }
                }
            }
        }
    }
    return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
}

Example 23 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class PlanFinalizer method preVisit.

@Override
public boolean preVisit(PlanNode visitable) {
    // if we come here again, prevent a further descend
    if (!this.allNodes.add(visitable)) {
        return false;
    }
    if (visitable instanceof SinkPlanNode) {
        this.sinks.add((SinkPlanNode) visitable);
    } else if (visitable instanceof SourcePlanNode) {
        this.sources.add((SourcePlanNode) visitable);
    } else if (visitable instanceof BinaryUnionPlanNode) {
        BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        if (unionNode.unionsStaticAndDynamicPath()) {
            unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
        }
    } else if (visitable instanceof BulkPartialSolutionPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
    } else if (visitable instanceof WorksetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    } else if (visitable instanceof SolutionSetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    }
    // one child candidate could have been referenced by multiple parents.
    for (Channel conn : visitable.getInputs()) {
        conn.setTarget(visitable);
        conn.getSource().addOutgoingChannel(conn);
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        c.setTarget(visitable);
        c.getSource().addOutgoingChannel(c);
    }
    // count the memory consumption
    this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
    for (Channel c : visitable.getInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    // pass the visitor to the iteraton's step function
    if (visitable instanceof IterationPlanNode) {
        // push the iteration node onto the stack
        final IterationPlanNode iterNode = (IterationPlanNode) visitable;
        this.stackOfIterationNodes.addLast(iterNode);
        // recurse
        ((IterationPlanNode) visitable).acceptForStepFunction(this);
        // pop the iteration node from the stack
        this.stackOfIterationNodes.removeLast();
    }
    return true;
}

Example 24 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class RangePartitionRewriter method postVisit.

@Override
public void postVisit(PlanNode node) {
    if (node instanceof IterationPlanNode) {
        IterationPlanNode iNode = (IterationPlanNode) node;
        if (!visitedIterationNodes.contains(iNode)) {
            visitedIterationNodes.add(iNode);
            iNode.acceptForStepFunction(this);
        }
    }
    final Iterable<Channel> inputChannels = node.getInputs();
    for (Channel channel : inputChannels) {
        ShipStrategyType shipStrategy = channel.getShipStrategy();
        // Make sure we only optimize the DAG for range partition, and do not optimize multi times.
        if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {
            if (channel.getDataDistribution() == null) {
                if (node.isOnDynamicPath()) {
                    throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
                }
                PlanNode channelSource = channel.getSource();
                List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
                channelSource.getOutgoingChannels().remove(channel);
                channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
            }
        }
    }
}

Example 25 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class AdditionalOperatorsTest method testCrossWithLarge.

@Test
public void testCrossWithLarge() {
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Long> set1 = env.generateSequence(0, 1);
    DataSet<Long> set2 = env.generateSequence(0, 1);
    set1.crossWithHuge(set2).name("Cross").output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    try {
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
        Channel in1 = crossPlanNode.getInput1();
        Channel in2 = crossPlanNode.getInput2();
        assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
    }
}