Examples with SingleInputPlanNode org.apache.flink.optimizer.plan.SingleInputPlanNode used on opensource projects

Example 56 with SingleInputPlanNode

use of org.apache.flink.optimizer.plan.SingleInputPlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testSingleInputOperatorsWithReCreation.

@Test
public void testSingleInputOperatorsWithReCreation() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        // set ship strategy in first channel, so later non matching global properties do not matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 5));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // set ship strategy in second channel, so previous non matching global properties void the match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 5));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // set local strategy in first channel, so later non matching local properties do not matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // set local strategy in second channel, so previous non matching local properties void the match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // create the properties on the same node as the requirement
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(5, 7));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(5, 7));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(5, 7), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with SingleInputPlanNode

use of org.apache.flink.optimizer.plan.SingleInputPlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testNoPartialSolutionFoundTwoInputOperator.

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
        SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
        SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
        Channel toMap1 = new Channel(source1);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(source2);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toJoin1 = new Channel(map1);
        Channel toJoin2 = new Channel(map2);
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
        assertEquals(NO_PARTIAL_SOLUTION, report);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 58 with SingleInputPlanNode

use of org.apache.flink.optimizer.plan.SingleInputPlanNode in project flink by apache.

the class UnionBetweenDynamicAndStaticPathTest method testUnionStaticSecond.

@Test
public void testUnionStaticSecond() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Long> input1 = env.generateSequence(1, 10);
        DataSet<Long> input2 = env.generateSequence(1, 10);
        IterativeDataSet<Long> iteration = input1.iterate(10);
        DataSet<Long> iterResult = iteration.closeWith(iteration.union(iteration).union(input2.union(input2)));
        iterResult.output(new DiscardingOutputFormat<Long>());
        iterResult.output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        assertEquals(2, op.getDataSinks().size());
        BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
        SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
        BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
        NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
        NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
        assertTrue(mixedUnion.unionsStaticAndDynamicPath());
        assertFalse(mixedUnion.getInput1().isOnDynamicPath());
        assertTrue(mixedUnion.getInput2().isOnDynamicPath());
        assertTrue(mixedUnion.getInput1().getTempMode().isCached());
        assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
        assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
        assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
        for (Channel c : staticUnion.getInputs()) {
            assertFalse(c.isOnDynamicPath());
        }
        for (Channel c : dynamicUnion.getInputs()) {
            assertTrue(c.isOnDynamicPath());
        }
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 59 with SingleInputPlanNode

use of org.apache.flink.optimizer.plan.SingleInputPlanNode in project flink by apache.

the class UnionBetweenDynamicAndStaticPathTest method testUnionStaticFirst.

@Test
public void testUnionStaticFirst() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Long> input1 = env.generateSequence(1, 10);
        DataSet<Long> input2 = env.generateSequence(1, 10);
        IterativeDataSet<Long> iteration = input1.iterate(10);
        DataSet<Long> result = iteration.closeWith(input2.union(input2).union(iteration.union(iteration)));
        result.output(new DiscardingOutputFormat<Long>());
        result.output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        assertEquals(2, op.getDataSinks().size());
        BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
        SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
        BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
        NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
        NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
        assertTrue(mixedUnion.unionsStaticAndDynamicPath());
        assertFalse(mixedUnion.getInput1().isOnDynamicPath());
        assertTrue(mixedUnion.getInput2().isOnDynamicPath());
        assertTrue(mixedUnion.getInput1().getTempMode().isCached());
        for (Channel c : staticUnion.getInputs()) {
            assertFalse(c.isOnDynamicPath());
        }
        for (Channel c : dynamicUnion.getInputs()) {
            assertTrue(c.isOnDynamicPath());
        }
        assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
        assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
        assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 60 with SingleInputPlanNode

use of org.apache.flink.optimizer.plan.SingleInputPlanNode in project flink by apache.

the class UnionPropertyPropagationTest method testUnion2.

@Test
public void testUnion2() {
    final int NUM_INPUTS = 4;
    // construct the plan it will be multiple flat maps, all unioned
    // and the "unioned" inputDataSet will be grouped
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> source = env.readTextFile(IN_FILE);
    DataSet<Tuple2<String, Integer>> lastUnion = source.flatMap(new DummyFlatMap());
    for (int i = 1; i < NUM_INPUTS; i++) {
        lastUnion = lastUnion.union(source.flatMap(new DummyFlatMap()));
    }
    DataSet<Tuple2<String, Integer>> result = lastUnion.groupBy(0).aggregate(Aggregations.SUM, 1);
    result.writeAsText(OUT_FILE);
    // return the plan
    Plan plan = env.createProgramPlan("Test union on new java-api");
    OptimizedPlan oPlan = compileNoStats(plan);
    JobGraphGenerator jobGen = new JobGraphGenerator();
    // Compile plan to verify that no error is thrown
    jobGen.compileJobGraph(oPlan);
    oPlan.accept(new Visitor<PlanNode>() {

        @Override
        public boolean preVisit(PlanNode visitable) {
            /* Test on the union output connections
				 * It must be under the GroupOperator and the strategy should be forward
				 */
            if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof GroupReduceOperatorBase) {
                final Channel inConn = ((SingleInputPlanNode) visitable).getInput();
                Assert.assertTrue("Union should just forward the Partitioning", inConn.getShipStrategy() == ShipStrategyType.FORWARD);
                Assert.assertTrue("Union Node should be under Group operator", inConn.getSource() instanceof NAryUnionPlanNode);
            }
            /* Test on the union input connections
				 * Must be NUM_INPUTS input connections, all FlatMapOperators with a own partitioning strategy(propably hash)
				 */
            if (visitable instanceof NAryUnionPlanNode) {
                int numberInputs = 0;
                for (Iterator<Channel> inputs = visitable.getInputs().iterator(); inputs.hasNext(); numberInputs++) {
                    final Channel inConn = inputs.next();
                    PlanNode inNode = inConn.getSource();
                    Assert.assertTrue("Input of Union should be FlatMapOperators", inNode.getProgramOperator() instanceof FlatMapOperatorBase);
                    Assert.assertTrue("Shipment strategy under union should partition the data", inConn.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
                }
                Assert.assertTrue("NAryUnion should have " + NUM_INPUTS + " inputs", numberInputs == NUM_INPUTS);
                return false;
            }
            return true;
        }

        @Override
        public void postVisit(PlanNode visitable) {
        // DO NOTHING
        }
    });
}