use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class PartitioningReusageTest method reuseBothPartitioningCoGroup4.
@Test
public void reuseBothPartitioningCoGroup4() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(0, 2).map(new MockMapper()).withForwardedFields("0;2").coGroup(set2.partitionByHash(1).map(new MockMapper()).withForwardedFields("1")).where(0, 2).equalTo(2, 1).with(new MockCoGroup());
coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileWithStats(plan);
SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
checkValidCoGroupInputProperties(coGroup);
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class PartitioningReusageTest method reuseSinglePartitioningCoGroup2.
@Test
public void reuseSinglePartitioningCoGroup2() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(0, 1).map(new MockMapper()).withForwardedFields("0;1").coGroup(set2).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileWithStats(plan);
SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
checkValidCoGroupInputProperties(coGroup);
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class PartitioningReusageTest method noPreviousPartitioningCoGroup2.
@Test
public void noPreviousPartitioningCoGroup2() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.coGroup(set2).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileWithStats(plan);
SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
checkValidCoGroupInputProperties(coGroup);
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class PartitioningReusageTest method reuseBothPartitioningCoGroup2.
@Test
public void reuseBothPartitioningCoGroup2() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(0, 1).map(new MockMapper()).withForwardedFields("0;1").coGroup(set2.partitionByHash(1, 2).map(new MockMapper()).withForwardedFields("1;2")).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileWithStats(plan);
SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
checkValidCoGroupInputProperties(coGroup);
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class PartitioningReusageTest method reuseBothPartitioningCoGroup3.
@Test
public void reuseBothPartitioningCoGroup3() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(0).map(new MockMapper()).withForwardedFields("0").coGroup(set2.partitionByHash(2, 1).map(new MockMapper()).withForwardedFields("2;1")).where(0, 1).equalTo(2, 1).with(new MockCoGroup());
coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileWithStats(plan);
SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
checkValidCoGroupInputProperties(coGroup);
}
Aggregations