Examples with DualInputPlanNode org.apache.flink.optimizer.plan.DualInputPlanNode used on opensource projects

Example 76 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningCoGroup3.

@Test
public void reuseSinglePartitioningCoGroup3() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.coGroup(set2.partitionByHash(2, 1).map(new MockMapper()).withForwardedFields("2;1")).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
    coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
    checkValidCoGroupInputProperties(coGroup);
}

Example 77 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class PartitioningReusageTest method reuseBothPartitioningJoin3.

@Test
public void reuseBothPartitioningJoin3() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.partitionByHash(0).map(new MockMapper()).withForwardedFields("0").join(set2.partitionByHash(2, 1).map(new MockMapper()).withForwardedFields("2;1"), JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 78 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class PartitioningReusageTest method reuseBothPartitioningCoGroup6.

@Test
public void reuseBothPartitioningCoGroup6() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(2).map(new MockMapper()).withForwardedFields("2").coGroup(set2.partitionByHash(2).map(new MockMapper()).withForwardedFields("2")).where(0, 2).equalTo(1, 2).with(new MockCoGroup());
    coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
    checkValidCoGroupInputProperties(coGroup);
}

Example 79 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningJoin3.

@Test
public void reuseSinglePartitioningJoin3() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.join(set2.partitionByHash(2, 1).map(new MockMapper()).withForwardedFields("2;1"), JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 80 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningJoin2.

@Test
public void reuseSinglePartitioningJoin2() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.partitionByHash(0, 1).map(new MockMapper()).withForwardedFields("0;1").join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}