use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class JoinCustomPartitioningTest method testJoinWithKeySelectors.
@Test
public void testJoinWithKeySelectors() {
try {
final Partitioner<Integer> partitioner = new TestPartitionerInt();
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
input1.join(input2, JoinHint.REPARTITION_HASH_FIRST).where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector()).withPartitioner(partitioner).output(new DiscardingOutputFormat<Tuple2<Pojo2, Pojo3>>());
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
SinkPlanNode sink = op.getDataSinks().iterator().next();
DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
assertEquals(partitioner, join.getInput1().getPartitioner());
assertEquals(partitioner, join.getInput2().getPartitioner());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class JoinTranslationTest method createPlanAndGetJoinNode.
private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Long> i1 = env.generateSequence(1, 1000);
DataSet<Long> i2 = env.generateSequence(1, 1000);
i1.join(i2, hint).where(new IdentityKeySelector<Long>()).equalTo(new IdentityKeySelector<Long>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
Plan plan = env.createProgramPlan();
// set statistics to the sources
plan.accept(new Visitor<Operator<?>>() {
@Override
public boolean preVisit(Operator<?> visitable) {
if (visitable instanceof GenericDataSourceBase) {
GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
setSourceStatistics(source, 10000000, 1000);
}
return true;
}
@Override
public void postVisit(Operator<?> visitable) {
}
});
OptimizedPlan op = compileWithStats(plan);
return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class JoinTranslationTest method testPartitionSortMergeTest.
@Test
public void testPartitionSortMergeTest() {
try {
DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_SORT_MERGE);
assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
assertEquals(DriverStrategy.INNER_MERGE, node.getDriverStrategy());
} catch (Exception e) {
e.printStackTrace();
fail(e.getClass().getSimpleName() + ": " + e.getMessage());
}
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class JoinTranslationTest method testOptimizerChoosesTest.
@Test
public void testOptimizerChoosesTest() {
try {
DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() || DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
} catch (Exception e) {
e.printStackTrace();
fail(e.getClass().getSimpleName() + ": " + e.getMessage());
}
}
use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.
the class JoinTranslationTest method testBroadcastHashFirstTest.
@Test
public void testBroadcastHashFirstTest() {
try {
DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST);
assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy());
assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
} catch (Exception e) {
e.printStackTrace();
fail(e.getClass().getSimpleName() + ": " + e.getMessage());
}
}
Aggregations