Examples with OptimizedPlan org.apache.flink.optimizer.plan.OptimizedPlan used on opensource projects

Example 71 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method reuseSinglePartitioningCoGroup1.

@Test
public void reuseSinglePartitioningCoGroup1() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(0, 1).map(new MockMapper()).withForwardedFields("0;1").coGroup(set2).where(0, 1).equalTo(0, 1).with(new MockCoGroup());
    coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
    checkValidCoGroupInputProperties(coGroup);
}

Example 72 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method noPreviousPartitioningJoin2.

@Test
public void noPreviousPartitioningJoin2() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> joined = set1.join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST).where(0, 1).equalTo(2, 1).with(new MockJoin());
    joined.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
    checkValidJoinInputProperties(join);
}

Example 73 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PartitioningReusageTest method reuseBothPartitioningCoGroup5.

@Test
public void reuseBothPartitioningCoGroup5() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
    DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1.partitionByHash(2).map(new MockMapper()).withForwardedFields("2").coGroup(set2.partitionByHash(1).map(new MockMapper()).withForwardedFields("1")).where(0, 2).equalTo(2, 1).with(new MockCoGroup());
    coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = compileWithStats(plan);
    SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
    DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
    checkValidCoGroupInputProperties(coGroup);
}

Example 74 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class PlanFinalizer method createFinalPlan.

public OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
    this.memoryConsumerWeights = 0;
    // traverse the graph
    for (SinkPlanNode node : sinks) {
        node.accept(this);
    }
    // assign the memory to each node
    if (this.memoryConsumerWeights > 0) {
        for (PlanNode node : this.allNodes) {
            // assign memory to the driver strategy of the node
            final int consumerWeight = node.getMemoryConsumerWeight();
            if (consumerWeight > 0) {
                final double relativeMem = (double) consumerWeight / this.memoryConsumerWeights;
                node.setRelativeMemoryPerSubtask(relativeMem);
                if (Optimizer.LOG.isDebugEnabled()) {
                    Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " + node.getProgramOperator().getName() + ".");
                }
            }
            // assign memory to the local and global strategies of the channels
            for (Channel c : node.getInputs()) {
                if (c.getLocalStrategy().dams()) {
                    final double relativeMem = 1.0 / this.memoryConsumerWeights;
                    c.setRelativeMemoryLocalStrategy(relativeMem);
                    if (Optimizer.LOG.isDebugEnabled()) {
                        Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " + "instance of " + c + ".");
                    }
                }
                if (c.getTempMode() != TempMode.NONE) {
                    final double relativeMem = 1.0 / this.memoryConsumerWeights;
                    c.setRelativeTempMemory(relativeMem);
                    if (Optimizer.LOG.isDebugEnabled()) {
                        Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " + "table for " + c + ".");
                    }
                }
            }
        }
    }
    return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
}

Example 75 with OptimizedPlan

use of org.apache.flink.optimizer.plan.OptimizedPlan in project flink by apache.

the class AdditionalOperatorsTest method testCrossWithLarge.

@Test
public void testCrossWithLarge() {
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Long> set1 = env.generateSequence(0, 1);
    DataSet<Long> set2 = env.generateSequence(0, 1);
    set1.crossWithHuge(set2).name("Cross").output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    try {
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
        Channel in1 = crossPlanNode.getInput1();
        Channel in2 = crossPlanNode.getInput2();
        assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
    }
}