Examples with JobGraphGenerator org.apache.flink.optimizer.plantranslate.JobGraphGenerator used on opensource projects

Example 51 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class NestedIterationsTest method testDeltaIterationInClosure.

@Test
public void testDeltaIterationInClosure() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> data1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DataSet<Tuple2<Long, Long>> data2 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> firstIteration = data1.iterateDelta(data1, 100, 0);
        DataSet<Tuple2<Long, Long>> inFirst = firstIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> firstResult = firstIteration.closeWith(inFirst, inFirst).map(new IdentityMapper<Tuple2<Long, Long>>());
        DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> mainIteration = data2.iterateDelta(data2, 100, 0);
        DataSet<Tuple2<Long, Long>> joined = mainIteration.getWorkset().join(firstResult).where(0).equalTo(0).projectFirst(0).projectSecond(0);
        DataSet<Tuple2<Long, Long>> mainResult = mainIteration.closeWith(joined, joined);
        mainResult.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        // optimizer should be able to translate this
        OptimizedPlan op = compileNoStats(p);
        // job graph generator should be able to translate this
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 52 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class BranchingPlansCompilerTest method testBranchEachContractType.

@SuppressWarnings("unchecked")
@Test
public void testBranchEachContractType() {
    try {
        // construct the plan
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Long> sourceA = env.generateSequence(0, 1);
        DataSet<Long> sourceB = env.generateSequence(0, 1);
        DataSet<Long> sourceC = env.generateSequence(0, 1);
        DataSet<Long> map1 = sourceA.map(new IdentityMapper<Long>()).name("Map 1");
        DataSet<Long> reduce1 = map1.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>()).name("Reduce 1");
        DataSet<Long> join1 = sourceB.union(sourceB).union(sourceC).join(sourceC).where("*").equalTo("*").with(new IdentityJoiner<Long>()).name("Join 1");
        DataSet<Long> coGroup1 = sourceA.coGroup(sourceB).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 1");
        DataSet<Long> cross1 = reduce1.cross(coGroup1).with(new IdentityCrosser<Long>()).name("Cross 1");
        DataSet<Long> coGroup2 = cross1.coGroup(cross1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 2");
        DataSet<Long> coGroup3 = map1.coGroup(join1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 3");
        DataSet<Long> map2 = coGroup3.map(new IdentityMapper<Long>()).name("Map 2");
        DataSet<Long> coGroup4 = map2.coGroup(join1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 4");
        DataSet<Long> coGroup5 = coGroup2.coGroup(coGroup1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 5");
        DataSet<Long> coGroup6 = reduce1.coGroup(coGroup4).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 6");
        DataSet<Long> coGroup7 = coGroup5.coGroup(coGroup6).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 7");
        coGroup7.union(sourceA).union(coGroup3).union(coGroup4).union(coGroup1).output(new DiscardingOutputFormat<Long>());
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        JobGraphGenerator jobGen = new JobGraphGenerator();
        // Compile plan to verify that no error is thrown
        jobGen.compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 53 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class BranchingPlansCompilerTest method testBranchingSourceMultipleTimes.

/**
 * <pre>
 *                              SINK
 *                               |
 *                            COGROUP
 *                        +---/    \----+
 *                       /               \
 *                      /             MATCH10
 *                     /               |    \
 *                    /                |  MATCH9
 *                MATCH5               |  |   \
 *                |   \                |  | MATCH8
 *                | MATCH4             |  |  |   \
 *                |  |   \             |  |  | MATCH7
 *                |  | MATCH3          |  |  |  |   \
 *                |  |  |   \          |  |  |  | MATCH6
 *                |  |  | MATCH2       |  |  |  |  |  |
 *                |  |  |  |   \       +--+--+--+--+--+
 *                |  |  |  | MATCH1            MAP
 *                \  |  |  |  |  | /-----------/
 *                (DATA SOURCE ONE)
 * </pre>
 */
@Test
public void testBranchingSourceMultipleTimes() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> mapped = source.map(new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {

            @Override
            public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
                return null;
            }
        });
        DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        joined5.coGroup(joined10).where(1).equalTo(1).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 54 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class BranchingPlansCompilerTest method testBranchingWithMultipleDataSinksSmall.

/**
 * <pre>
 *             (SRC A)
 *             /     \
 *        (SINK A)    (SINK B)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinksSmall() {
    try {
        String outPath1 = "/tmp/out1";
        String outPath2 = "/tmp/out2";
        // construct the plan
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Long> source1 = env.generateSequence(0, 1);
        source1.writeAsText(outPath1);
        source1.writeAsText(outPath2);
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        // ---------- check the optimizer plan ----------
        // number of sinks
        Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
        // sinks contain all sink paths
        Set<String> allSinks = new HashSet<String>();
        allSinks.add(outPath1);
        allSinks.add(outPath2);
        for (SinkPlanNode n : oPlan.getDataSinks()) {
            String path = ((TextOutputFormat<String>) n.getSinkNode().getOperator().getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString();
            Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
        }
        // ---------- compile plan to job graph to verify that no error is thrown ----------
        JobGraphGenerator jobGen = new JobGraphGenerator();
        jobGen.compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 55 with JobGraphGenerator

use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.

the class CachedMatchStrategyCompilerTest method testRightSideCountercheck.

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached
 * variant
 */
@Test
public void testRightSideCountercheck() {
    try {
        Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
        assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}