Examples with OptimizedPlan org.apache.flink.optimizer.plan.OptimizedPlan used on opensource projects

Example 46 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method reuseBothPartitioningJoin5.

@Test
public void reuseBothPartitioningJoin5() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.partitionByHash(2).map(new MockMapper()).withForwardedFields("2").join(set2.partitionByHash(1).map(new MockMapper()).withForwardedFields("1"), JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 2).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 47 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningCoGroup5.

@Test
public void reuseSinglePartitioningCoGroup5() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.coGroup(set2.partitionByHash(2).map(new MockMapper()).withForwardedFields("2")).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
    coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
    checkValidCoGroupInputProperties(coGroup);
}

Example 48 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method noPreviousPartitioningJoin1.

@Test
public void noPreviousPartitioningJoin1() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(0, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 49 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningJoin5.

@Test
public void reuseSinglePartitioningJoin5() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.join(set2.partitionByHash(2).map(new MockMapper()).withForwardedFields("2"), JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 50 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PipelineBreakerTest method testPipelineBreakerBroadcastedPartialSolution.

/**
	 * 
	 * 
	 * 
	 * <pre>
	 *                                +----------- ITERATION ---------+
	 *                                |                               |
	 *                               +--+                           +----+
	 *  (source 1) ----------------->|PS| ------------ +        +-->|next|---> (sink)
	 *                               +--+              | (BC)   |   +----+
	 *                                |                V        |     |
	 *  (source 2) --> (map) --+------|-----------> (MAPPER) ---+     |
	 *                         |      |                ^              |
	 *                         |      |                | (BC)         |
	 *                         |      +----------------|--------------+
	 *                         |                       |
	 *                         +--(map) --> (reduce) --+
	 * </pre>
	 */
@Test
public void testPipelineBreakerBroadcastedPartialSolution() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
        env.setParallelism(64);
        DataSet<Long> initialSource = env.generateSequence(1, 10);
        IterativeDataSet<Long> iteration = initialSource.iterate(100);
        DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
        DataSet<Long> bcInput1 = sourceWithMapper.map(new IdentityMapper<Long>()).reduce(new SelectOneReducer<Long>());
        DataSet<Long> result = sourceWithMapper.map(new IdentityMapper<Long>()).withBroadcastSet(iteration, "bc2").withBroadcastSet(bcInput1, "bc1");
        iteration.closeWith(result).output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        BulkIterationPlanNode iterationPlanNode = (BulkIterationPlanNode) sink.getInput().getSource();
        SingleInputPlanNode mapper = (SingleInputPlanNode) iterationPlanNode.getRootOfStepFunction();
        assertEquals(TempMode.CACHED, mapper.getInput().getTempMode());
        assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}