Examples with PlanJSONDumpGenerator org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator used on opensource projects

Example 6 with PlanJSONDumpGenerator

use of org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator in project flink by apache.

the class ConnectedComponentsCoGroupTest method testWorksetConnectedComponents.

@Test
public void testWorksetConnectedComponents() {
    Plan plan = getConnectedComponentsCoGroupPlan();
    plan.setExecutionConfig(new ExecutionConfig());
    OptimizedPlan optPlan = compileNoStats(plan);
    OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);
    if (PRINT_PLAN) {
        PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
        String json = dumper.getOptimizerPlanAsJSON(optPlan);
        System.out.println(json);
    }
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE);
    // --------------------------------------------------------------------
    // Plan validation:
    //
    // We expect the plan to go with a sort-merge join, because the CoGroup
    // sorts and the join in the successive iteration can re-exploit the sorting.
    // --------------------------------------------------------------------
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.INNER_MERGE, neighborsJoin.getDriverStrategy());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
    Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy());
    Assert.assertEquals(set0, cogroup.getKeysForInput1());
    Assert.assertEquals(set0, cogroup.getKeysForInput2());
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
    // workset
    Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy());
    // edges
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy());
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
    Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached());
    // min id
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy());
    // solution set
    Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy());
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
    // the sort for the neighbor join in the first iteration is pushed out of the loop
    Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy());
    // workset
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy());
    // edges
    Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy());
    // solution set
    Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy());
    // check the caches
    Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode());
    JobGraphGenerator jgg = new JobGraphGenerator();
    jgg.compileJobGraph(optPlan);
}

Example 7 with PlanJSONDumpGenerator

use of org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator in project flink by apache.

the class TestEnvironment method getExecutionPlan.

@Override
public String getExecutionPlan() throws Exception {
    OptimizedPlan op = compileProgram("unused");
    PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
    return jsonGen.getOptimizerPlanAsJSON(op);
}

Example 8 with PlanJSONDumpGenerator

use of org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator in project flink by apache.

the class IterationsCompilerTest method testSolutionSetDeltaDependsOnBroadcastVariable.

@Test
public void testSolutionSetDeltaDependsOnBroadcastVariable() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
        DataSet<Tuple2<Long, Long>> invariantInput = env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
        // iteration from here
        DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = source.iterateDelta(source, 1000, 1);
        DataSet<Tuple2<Long, Long>> result = invariantInput.map(new IdentityMapper<Tuple2<Long, Long>>()).withBroadcastSet(iter.getWorkset(), "bc data").join(iter.getSolutionSet()).where(0).equalTo(1).projectFirst(1).projectSecond(1);
        iter.closeWith(result.map(new IdentityMapper<Tuple2<Long, Long>>()), result).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        OptimizedPlan p = compileNoStats(env.createProgramPlan());
        // check that the JSON generator accepts this plan
        new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(p);
        // check that the JobGraphGenerator accepts the plan
        new JobGraphGenerator().compileJobGraph(p);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with PlanJSONDumpGenerator

use of org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator in project flink by apache.

the class LocalExecutor method getOptimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 *
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), this.configuration);
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

Example 10 with PlanJSONDumpGenerator

use of org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator in project flink by apache.

the class LocalExecutor method optimizerPlanAsJSON.

/**
	 * Creates a JSON representation of the given dataflow's execution plan.
	 * 
	 * @param plan The dataflow plan.
	 * @return The dataflow's execution plan, as a JSON string.
	 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
	 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}