Examples with DualInputPlanNode org.apache.flink.optimizer.plan.DualInputPlanNode used on opensource projects

Example 11 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testTwoOperatorsBothDependent.

@Test
public void testTwoOperatorsBothDependent() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
        Channel toMap1 = new Channel(target);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(target);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toJoin1 = new Channel(map1);
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        Channel toJoin2 = new Channel(map2);
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
        Channel toAfterJoin = new Channel(join);
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
        // no properties from the partial solution, no required properties
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.EMPTY;
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some properties from the partial solution, no required properties
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on one input and met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on both input and met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            toJoin2.setRequiredGlobalProps(rgp);
            toJoin2.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on both inputs, one not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
            rgp1.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp1 = new RequestedLocalProperties();
            rlp1.setGroupedFields(new FieldList(2, 1));
            RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
            rgp2.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp2 = new RequestedLocalProperties();
            rlp2.setGroupedFields(new FieldList(0, 3));
            toJoin1.setRequiredGlobalProps(rgp1);
            toJoin1.setRequiredLocalProps(rlp1);
            toJoin2.setRequiredGlobalProps(rgp2);
            toJoin2.setRequiredLocalProps(rlp2);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // test override on both inputs, later requirement ignored
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(0, 3));
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin2.setRequiredGlobalProps(null);
            toJoin2.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // test override on one inputs, later requirement met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(PENDING, report);
        }
        // test override on one input, later requirement not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(3));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(77, 69));
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // test override on one input locally, later global requirement not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(3));
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(3), new boolean[] { false });
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testTwoOperatorsOneIndependent.

@Test
public void testTwoOperatorsOneIndependent() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
        SourcePlanNode source = new SourcePlanNode(getSourceNode(), "Other Source");
        Channel toMap1 = new Channel(target);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(source);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toJoin1 = new Channel(map1);
        Channel toJoin2 = new Channel(map2);
        DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
        Channel toAfterJoin = new Channel(join);
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
        // attach some properties to the non-relevant input
        {
            toMap2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] { true, true });
            RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
            joinGp.setFullyReplicated();
            RequestedLocalProperties joinLp = new RequestedLocalProperties();
            joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin2.setLocalStrategy(LocalStrategy.NONE);
            toJoin2.setRequiredGlobalProps(joinGp);
            toJoin2.setRequiredLocalProps(joinLp);
        }
        // ------------------------------------------------------------------------------------
        // no properties from the partial solution, no required properties
        {
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.EMPTY;
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some properties from the partial solution, no required properties
        {
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // produced properties match relevant input
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // produced properties do not match relevant input
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(1, 2, 3));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // produced properties overridden before join
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(rgp);
            toMap1.setRequiredLocalProps(rlp);
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] { true, false });
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // produced properties before join match, after join match as well
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // produced properties before join match, after join do not match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
            rgp1.setHashPartitioned(new FieldList(0));
            RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
            rgp2.setHashPartitioned(new FieldList(3));
            RequestedLocalProperties rlp1 = new RequestedLocalProperties();
            rlp1.setGroupedFields(new FieldList(2, 1));
            RequestedLocalProperties rlp2 = new RequestedLocalProperties();
            rlp2.setGroupedFields(new FieldList(3, 4));
            toJoin1.setRequiredGlobalProps(rgp1);
            toJoin1.setRequiredLocalProps(rlp1);
            toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
            toAfterJoin.setRequiredGlobalProps(rgp2);
            toAfterJoin.setRequiredLocalProps(rlp2);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // produced properties are overridden, does not matter that they do not match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setAnyPartitioning(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(1, 2, 3));
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] { true, false });
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // local property overridden before join, local property mismatch after join not relevant
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setAnyPartitioning(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(1, 2, 3));
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] { true, false });
            toAfterJoin.setRequiredGlobalProps(null);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // local property overridden before join, global property mismatch after join void the match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setAnyPartitioning(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(1, 2, 3));
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] { true, false });
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 13 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class ParallelismChangeTest method checkPropertyHandlingWithTwoInputs.

/**
	 * Checks that re-partitioning happens when the inputs of a two-input contract have different parallelisms.
	 * 
	 * Test Plan:
	 * <pre>
	 * 
	 * (source) -> reduce -\
	 *                      Match -> (sink)
	 * (source) -> reduce -/
	 * 
	 * </pre>
	 * 
	 */
@Test
public void checkPropertyHandlingWithTwoInputs() {
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Long> set1 = env.generateSequence(0, 1).setParallelism(5);
    DataSet<Long> set2 = env.generateSequence(0, 1).setParallelism(7);
    DataSet<Long> reduce1 = set1.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>()).withForwardedFields("*").setParallelism(5);
    DataSet<Long> reduce2 = set2.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>()).withForwardedFields("*").setParallelism(7);
    reduce1.join(reduce2).where("*").equalTo("*").with(new IdentityJoiner<Long>()).setParallelism(5).output(new DiscardingOutputFormat<Long>()).setParallelism(5);
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    JobGraphGenerator jobGen = new JobGraphGenerator();
    //Compile plan to verify that no error is thrown
    jobGen.compileJobGraph(oPlan);
    oPlan.accept(new Visitor<PlanNode>() {

        @Override
        public boolean preVisit(PlanNode visitable) {
            if (visitable instanceof DualInputPlanNode) {
                DualInputPlanNode node = (DualInputPlanNode) visitable;
                Channel c1 = node.getInput1();
                Channel c2 = node.getInput2();
                Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy());
                Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy());
                return false;
            }
            return true;
        }

        @Override
        public void postVisit(PlanNode visitable) {
        // DO NOTHING
        }
    });
}

Example 14 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class ReplicatingDataSourceTest method checkCrossWithReplicatedSourceInput.

/**
	 * Tests cross program with replicated data source.
	 */
@Test
public void checkCrossWithReplicatedSourceInput() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
    ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif = new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));
    DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
    DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
    DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1.cross(source2).writeAsText("/some/newpath");
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    // when cross should have forward strategy on both sides
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();
    ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
    ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
}

Example 15 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class ReplicatingDataSourceTest method checkJoinWithReplicatedSourceInputBehindMapPartition.

/**
	 * Tests join program with replicated data source behind map partition.
	 */
@Test
public void checkJoinWithReplicatedSourceInputBehindMapPartition() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
    ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif = new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));
    DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
    DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
    DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1.mapPartition(new IdPMap()).join(source2).where("*").equalTo("*").writeAsText("/some/newpath");
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    // when join should have forward strategy on both sides
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
    ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
    ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}