Examples with BinaryUnionPlanNode org.apache.flink.optimizer.plan.BinaryUnionPlanNode used on opensource projects

Example 1 with BinaryUnionPlanNode

use of org.apache.flink.optimizer.plan.BinaryUnionPlanNode in project flink by apache.

the class BinaryUnionReplacer method postVisit.

@Override
public void postVisit(PlanNode visitable) {
    if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
        if (!unionNode.unionsStaticAndDynamicPath()) {
            // both on static path, or both on dynamic path. we can collapse them
            NAryUnionPlanNode newUnionNode;
            List<Channel> inputs = new ArrayList<Channel>();
            collect(in1, inputs);
            collect(in2, inputs);
            newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
            newUnionNode.setParallelism(unionNode.getParallelism());
            for (Channel c : inputs) {
                c.setTarget(newUnionNode);
            }
            for (Channel channel : unionNode.getOutgoingChannels()) {
                channel.swapUnionNodes(newUnionNode);
                newUnionNode.addOutgoingChannel(channel);
            }
        } else {
            // make sure that the first input is the cached (static) and the second input is the dynamic
            if (in1.isOnDynamicPath()) {
                BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
                in1.setTarget(newUnionNode);
                in2.setTarget(newUnionNode);
                for (Channel channel : unionNode.getOutgoingChannels()) {
                    channel.swapUnionNodes(newUnionNode);
                    newUnionNode.addOutgoingChannel(channel);
                }
            }
        }
    }
}

Example 2 with BinaryUnionPlanNode

use of org.apache.flink.optimizer.plan.BinaryUnionPlanNode in project flink by apache.

the class PlanFinalizer method preVisit.

@Override
public boolean preVisit(PlanNode visitable) {
    // if we come here again, prevent a further descend
    if (!this.allNodes.add(visitable)) {
        return false;
    }
    if (visitable instanceof SinkPlanNode) {
        this.sinks.add((SinkPlanNode) visitable);
    } else if (visitable instanceof SourcePlanNode) {
        this.sources.add((SourcePlanNode) visitable);
    } else if (visitable instanceof BinaryUnionPlanNode) {
        BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        if (unionNode.unionsStaticAndDynamicPath()) {
            unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
        }
    } else if (visitable instanceof BulkPartialSolutionPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
    } else if (visitable instanceof WorksetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    } else if (visitable instanceof SolutionSetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    }
    // one child candidate could have been referenced by multiple parents.
    for (Channel conn : visitable.getInputs()) {
        conn.setTarget(visitable);
        conn.getSource().addOutgoingChannel(conn);
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        c.setTarget(visitable);
        c.getSource().addOutgoingChannel(c);
    }
    // count the memory consumption
    this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
    for (Channel c : visitable.getInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    // pass the visitor to the iteraton's step function
    if (visitable instanceof IterationPlanNode) {
        // push the iteration node onto the stack
        final IterationPlanNode iterNode = (IterationPlanNode) visitable;
        this.stackOfIterationNodes.addLast(iterNode);
        // recurse
        ((IterationPlanNode) visitable).acceptForStepFunction(this);
        // pop the iteration node from the stack
        this.stackOfIterationNodes.removeLast();
    }
    return true;
}

Example 3 with BinaryUnionPlanNode

use of org.apache.flink.optimizer.plan.BinaryUnionPlanNode in project flink by apache.

the class UnionBetweenDynamicAndStaticPathTest method testUnionStaticSecond.

@Test
public void testUnionStaticSecond() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Long> input1 = env.generateSequence(1, 10);
        DataSet<Long> input2 = env.generateSequence(1, 10);
        IterativeDataSet<Long> iteration = input1.iterate(10);
        DataSet<Long> iterResult = iteration.closeWith(iteration.union(iteration).union(input2.union(input2)));
        iterResult.output(new DiscardingOutputFormat<Long>());
        iterResult.output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        assertEquals(2, op.getDataSinks().size());
        BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
        SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
        BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
        NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
        NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
        assertTrue(mixedUnion.unionsStaticAndDynamicPath());
        assertFalse(mixedUnion.getInput1().isOnDynamicPath());
        assertTrue(mixedUnion.getInput2().isOnDynamicPath());
        assertTrue(mixedUnion.getInput1().getTempMode().isCached());
        assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
        assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
        assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
        for (Channel c : staticUnion.getInputs()) {
            assertFalse(c.isOnDynamicPath());
        }
        for (Channel c : dynamicUnion.getInputs()) {
            assertTrue(c.isOnDynamicPath());
        }
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with BinaryUnionPlanNode

use of org.apache.flink.optimizer.plan.BinaryUnionPlanNode in project flink by apache.

the class UnionBetweenDynamicAndStaticPathTest method testUnionStaticFirst.

@Test
public void testUnionStaticFirst() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Long> input1 = env.generateSequence(1, 10);
        DataSet<Long> input2 = env.generateSequence(1, 10);
        IterativeDataSet<Long> iteration = input1.iterate(10);
        DataSet<Long> result = iteration.closeWith(input2.union(input2).union(iteration.union(iteration)));
        result.output(new DiscardingOutputFormat<Long>());
        result.output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        assertEquals(2, op.getDataSinks().size());
        BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
        SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
        BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
        NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
        NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
        assertTrue(mixedUnion.unionsStaticAndDynamicPath());
        assertFalse(mixedUnion.getInput1().isOnDynamicPath());
        assertTrue(mixedUnion.getInput2().isOnDynamicPath());
        assertTrue(mixedUnion.getInput1().getTempMode().isCached());
        for (Channel c : staticUnion.getInputs()) {
            assertFalse(c.isOnDynamicPath());
        }
        for (Channel c : dynamicUnion.getInputs()) {
            assertTrue(c.isOnDynamicPath());
        }
        assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
        assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
        assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}