Search in sources :

Example 86 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class JoinCustomPartitioningTest method testJoinWithKeySelectors.

@Test
public void testJoinWithKeySelectors() {
    try {
        final Partitioner<Integer> partitioner = new TestPartitionerInt();
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
        DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
        input1.join(input2, JoinHint.REPARTITION_HASH_FIRST).where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector()).withPartitioner(partitioner).output(new DiscardingOutputFormat<Tuple2<Pojo2, Pojo3>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
        assertEquals(partitioner, join.getInput1().getPartitioner());
        assertEquals(partitioner, join.getInput2().getPartitioner());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Plan(org.apache.flink.api.common.Plan) OptimizedPlan(org.apache.flink.optimizer.plan.OptimizedPlan) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) OptimizedPlan(org.apache.flink.optimizer.plan.OptimizedPlan) DualInputPlanNode(org.apache.flink.optimizer.plan.DualInputPlanNode) Tuple2(org.apache.flink.api.java.tuple.Tuple2) SinkPlanNode(org.apache.flink.optimizer.plan.SinkPlanNode) Test(org.junit.Test)

Example 87 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class JoinTranslationTest method createPlanAndGetJoinNode.

private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Long> i1 = env.generateSequence(1, 1000);
    DataSet<Long> i2 = env.generateSequence(1, 1000);
    i1.join(i2, hint).where(new IdentityKeySelector<Long>()).equalTo(new IdentityKeySelector<Long>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    Plan plan = env.createProgramPlan();
    // set statistics to the sources
    plan.accept(new Visitor<Operator<?>>() {

        @Override
        public boolean preVisit(Operator<?> visitable) {
            if (visitable instanceof GenericDataSourceBase) {
                GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
                setSourceStatistics(source, 10000000, 1000);
            }
            return true;
        }

        @Override
        public void postVisit(Operator<?> visitable) {
        }
    });
    OptimizedPlan op = compileWithStats(plan);
    return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
}
Also used : Operator(org.apache.flink.api.common.operators.Operator) ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Plan(org.apache.flink.api.common.Plan) OptimizedPlan(org.apache.flink.optimizer.plan.OptimizedPlan) OptimizedPlan(org.apache.flink.optimizer.plan.OptimizedPlan) DualInputPlanNode(org.apache.flink.optimizer.plan.DualInputPlanNode) GenericDataSourceBase(org.apache.flink.api.common.operators.GenericDataSourceBase) Tuple2(org.apache.flink.api.java.tuple.Tuple2) SinkPlanNode(org.apache.flink.optimizer.plan.SinkPlanNode)

Example 88 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class JoinTranslationTest method testPartitionSortMergeTest.

@Test
public void testPartitionSortMergeTest() {
    try {
        DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_SORT_MERGE);
        assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
        assertEquals(DriverStrategy.INNER_MERGE, node.getDriverStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + ": " + e.getMessage());
    }
}
Also used : DualInputPlanNode(org.apache.flink.optimizer.plan.DualInputPlanNode) Test(org.junit.Test)

Example 89 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class JoinTranslationTest method testOptimizerChoosesTest.

@Test
public void testOptimizerChoosesTest() {
    try {
        DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
        assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
        assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() || DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + ": " + e.getMessage());
    }
}
Also used : DualInputPlanNode(org.apache.flink.optimizer.plan.DualInputPlanNode) Test(org.junit.Test)

Example 90 with DualInputPlanNode

use of org.apache.flink.optimizer.plan.DualInputPlanNode in project flink by apache.

the class JoinTranslationTest method testBroadcastHashFirstTest.

@Test
public void testBroadcastHashFirstTest() {
    try {
        DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST);
        assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy());
        assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + ": " + e.getMessage());
    }
}
Also used : DualInputPlanNode(org.apache.flink.optimizer.plan.DualInputPlanNode) Test(org.junit.Test)

Aggregations

DualInputPlanNode (org.apache.flink.optimizer.plan.DualInputPlanNode)96 Test (org.junit.Test)86 OptimizedPlan (org.apache.flink.optimizer.plan.OptimizedPlan)81 Plan (org.apache.flink.api.common.Plan)76 SinkPlanNode (org.apache.flink.optimizer.plan.SinkPlanNode)67 ExecutionEnvironment (org.apache.flink.api.java.ExecutionEnvironment)65 Tuple3 (org.apache.flink.api.java.tuple.Tuple3)36 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)31 SingleInputPlanNode (org.apache.flink.optimizer.plan.SingleInputPlanNode)27 JobGraphGenerator (org.apache.flink.optimizer.plantranslate.JobGraphGenerator)19 Channel (org.apache.flink.optimizer.plan.Channel)14 WorksetIterationPlanNode (org.apache.flink.optimizer.plan.WorksetIterationPlanNode)13 FieldList (org.apache.flink.api.common.operators.util.FieldList)12 InvalidProgramException (org.apache.flink.api.common.InvalidProgramException)11 DiscardingOutputFormat (org.apache.flink.api.java.io.DiscardingOutputFormat)11 PlanNode (org.apache.flink.optimizer.plan.PlanNode)11 Tuple1 (org.apache.flink.api.java.tuple.Tuple1)10 SourcePlanNode (org.apache.flink.optimizer.plan.SourcePlanNode)10 ShipStrategyType (org.apache.flink.runtime.operators.shipping.ShipStrategyType)10 ReplicatingInputFormat (org.apache.flink.api.common.io.ReplicatingInputFormat)8