Examples with JobGraphGenerator org.apache.flink.optimizer.plantranslate.JobGraphGenerator used on opensource projects

Example 16 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class WorksetIterationsRecordApiCompilerTest method testRecordApiWithDeferredSoltionSetUpdateWithMapper.

@Test
public void testRecordApiWithDeferredSoltionSetUpdateWithMapper() {
    Plan plan = getTestPlan(false, true);
    OptimizedPlan oPlan;
    try {
        oPlan = compileNoStats(plan);
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
        // silence the compiler
        return;
    }
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop, 
    // the in-loop partitioning is before the final reducer
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
    // currently, the system may partition before or after the mapper
    ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
    ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
    assertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH));
    new JobGraphGenerator().compileJobGraph(oPlan);
}

Example 17 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class CachedMatchStrategyCompilerTest method testLeftSide.

/**
	 * This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
	 * when inside of an iteration an on the static path
	 */
@Test
public void testLeftSide() {
    try {
        Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
        assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

Example 18 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class CachedMatchStrategyCompilerTest method testLeftSideCountercheck.

/**
	 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
	 */
@Test
public void testLeftSideCountercheck() {
    try {
        Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
        assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
        assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

Example 19 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class CachedMatchStrategyCompilerTest method testCorrectChoosing.

/**
	 * This test simulates a join of a big left side with a small right side inside of an iteration, where the small side is on a static path.
	 * Currently the best execution plan is a HYBRIDHASH_BUILD_SECOND_CACHED, where the small side is hashed and cached.
	 * This test also makes sure that all relevant plans are correctly enumerated by the optimizer.
	 */
@Test
public void testCorrectChoosing() {
    try {
        Plan plan = getTestPlanRightStatic("");
        SourceCollectorVisitor sourceCollector = new SourceCollectorVisitor();
        plan.accept(sourceCollector);
        for (GenericDataSourceBase<?, ?> s : sourceCollector.getSources()) {
            if (s.getName().equals("bigFile")) {
                this.setSourceStatistics(s, 10000000, 1000);
            } else if (s.getName().equals("smallFile")) {
                this.setSourceStatistics(s, 100, 100);
            }
        }
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
        assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

Example 20 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class BranchingPlansCompilerTest method testBranchingWithMultipleDataSinks.

/**
	 * 
	 * <pre>

	 *              (SINK A)
	 *                  |    (SINK B)    (SINK C)
	 *                CROSS    /          /
	 *               /     \   |  +------+
	 *              /       \  | /
	 *          REDUCE      MATCH2
	 *             |    +---/    \
	 *              \  /          |
	 *               MAP          |
	 *                |           |
	 *             COGROUP      MATCH1
	 *             /     \     /     \
	 *        (SRC A)    (SRC B)    (SRC C)
	 * </pre>
	 */
@Test
public void testBranchingWithMultipleDataSinks() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB).where(0).equalTo(1).with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {

            @Override
            public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) {
            }
        }).map(new IdentityMapper<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC).where(0).equalTo(1).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined).where(1).equalTo(1).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> reduced = mapped.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());
        reduced.cross(joined2).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
        joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}