Examples with SinkPlanNode org.apache.flink.optimizer.plan.SinkPlanNode used on opensource projects

Example 71 with SinkPlanNode

use of org.apache.flink.optimizer.plan.SinkPlanNode in project flink by apache.

the class SpargelCompilerTest method testSpargelCompiler.

@SuppressWarnings("serial")
@Test
public void testSpargelCompiler() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // compose test program
    DataSet<Vertex<Long, Long>> initialVertices = env.fromElements(new Tuple2<>(1L, 1L), new Tuple2<>(2L, 2L)).map(new Tuple2ToVertexMap<>());
    DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple2<>(1L, 2L)).map(new MapFunction<Tuple2<Long, Long>, Edge<Long, NullValue>>() {

        public Edge<Long, NullValue> map(Tuple2<Long, Long> edge) {
            return new Edge<>(edge.f0, edge.f1, NullValue.getInstance());
        }
    });
    Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);
    DataSet<Vertex<Long, Long>> result = graph.runScatterGatherIteration(new ConnectedComponents.CCMessenger<>(BasicTypeInfo.LONG_TYPE_INFO), new ConnectedComponents.CCUpdater<>(), 100).getVertices();
    result.output(new DiscardingOutputFormat<>());
    Plan p = env.createProgramPlan("Spargel Connected Components");
    OptimizedPlan op = compileNoStats(p);
    // check the sink
    SinkPlanNode sink = op.getDataSinks().iterator().next();
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
    // check the iteration
    WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
    // check the solution set join and the delta
    PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
    assertTrue(ssDelta instanceof // this is only true if the update functions preserves
    DualInputPlanNode);
    // the partitioning
    DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
    assertEquals(DEFAULT_PARALLELISM, ssJoin.getParallelism());
    assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
    assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
    // check the workset set join
    DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
    assertEquals(DEFAULT_PARALLELISM, edgeJoin.getParallelism());
    assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
    assertTrue(edgeJoin.getInput1().getTempMode().isCached());
    assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
    // check that the initial partitioning is pushed out of the loop
    assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
    assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
    assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
    // check that the initial workset sort is outside the loop
    assertEquals(LocalStrategy.SORT, iteration.getInput2().getLocalStrategy());
    assertEquals(new FieldList(0), iteration.getInput2().getLocalStrategyKeys());
}

Example 72 with SinkPlanNode

use of org.apache.flink.optimizer.plan.SinkPlanNode in project flink by apache.

the class DataSinkNode method getAlternativePlans.

// --------------------------------------------------------------------------------------------
// Recursive Optimization
// --------------------------------------------------------------------------------------------
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    // calculate alternative sub-plans for predecessor
    List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    List<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final int parallelism = getParallelism();
    final int inDop = getPredecessorNode().getParallelism();
    final ExecutionMode executionMode = this.input.getDataExchangeMode();
    final boolean dopChange = parallelism != inDop;
    final boolean breakPipeline = this.input.isBreakingPipeline();
    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
        for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
            for (RequestedLocalProperties lp : ips.getLocalProperties()) {
                Channel c = new Channel(p);
                gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
                lp.parameterizeChannel(c);
                c.setRequiredLocalProps(lp);
                c.setRequiredGlobalProps(gp);
                // no need to check whether the created properties meet what we need in case
                // of ordering or global ordering, because the only interesting properties we
                // have
                // are what we require
                outputPlans.add(new SinkPlanNode(this, "DataSink (" + this.getOperator().getName() + ")", c));
            }
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    this.cachedPlans = outputPlans;
    return outputPlans;
}

Example 73 with SinkPlanNode

use of org.apache.flink.optimizer.plan.SinkPlanNode in project flink by apache.

the class ConnectedComponentsTest method testWorksetConnectedComponentsWithSolutionSetAsFirstInput.

@Test
public void testWorksetConnectedComponentsWithSolutionSetAsFirstInput() {
    Plan plan = getConnectedComponentsPlan(DEFAULT_PARALLELISM, 100, true);
    OptimizedPlan optPlan = compileNoStats(plan);
    OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    SingleInputPlanNode minIdReducer = or.getNode(MIN_ID_REDUCER);
    SingleInputPlanNode minIdCombiner = (SingleInputPlanNode) minIdReducer.getPredecessor();
    DualInputPlanNode updatingMatch = or.getNode(UPDATE_ID_MATCH);
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, neighborsJoin.getDriverStrategy());
    Assert.assertTrue(!neighborsJoin.getInput1().getTempMode().isCached());
    Assert.assertTrue(!neighborsJoin.getInput2().getTempMode().isCached());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, updatingMatch.getDriverStrategy());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput1());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput2());
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, // workset
    neighborsJoin.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, // edges
    neighborsJoin.getInput2().getShipStrategy());
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, minIdReducer.getInput().getShipStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, minIdCombiner.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.FORWARD, // solution set
    updatingMatch.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.FORWARD, // min id
    updatingMatch.getInput2().getShipStrategy());
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // workset
    neighborsJoin.getInput1().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // edges
    neighborsJoin.getInput2().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.COMBININGSORT, minIdReducer.getInput().getLocalStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getLocalStrategyKeys());
    Assert.assertEquals(LocalStrategy.NONE, minIdCombiner.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // min id
    updatingMatch.getInput1().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // solution set
    updatingMatch.getInput2().getLocalStrategy());
    // check the dams
    Assert.assertEquals(TempMode.NONE, iter.getInitialWorksetInput().getTempMode());
    Assert.assertEquals(TempMode.NONE, iter.getInitialSolutionSetInput().getTempMode());
    Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialWorksetInput().getDataExchangeMode());
    Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialSolutionSetInput().getDataExchangeMode());
    JobGraphGenerator jgg = new JobGraphGenerator();
    jgg.compileJobGraph(optPlan);
}

Example 74 with SinkPlanNode

use of org.apache.flink.optimizer.plan.SinkPlanNode in project flink by apache.

the class ConnectedComponentsTest method testWorksetConnectedComponents.

@Test
public void testWorksetConnectedComponents() {
    Plan plan = getConnectedComponentsPlan(DEFAULT_PARALLELISM, 100, false);
    OptimizedPlan optPlan = compileNoStats(plan);
    OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    SingleInputPlanNode minIdReducer = or.getNode(MIN_ID_REDUCER);
    SingleInputPlanNode minIdCombiner = (SingleInputPlanNode) minIdReducer.getPredecessor();
    DualInputPlanNode updatingMatch = or.getNode(UPDATE_ID_MATCH);
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, neighborsJoin.getDriverStrategy());
    Assert.assertTrue(!neighborsJoin.getInput1().getTempMode().isCached());
    Assert.assertTrue(!neighborsJoin.getInput2().getTempMode().isCached());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, updatingMatch.getDriverStrategy());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput1());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput2());
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, // workset
    neighborsJoin.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, // edges
    neighborsJoin.getInput2().getShipStrategy());
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, minIdReducer.getInput().getShipStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, minIdCombiner.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.FORWARD, // min id
    updatingMatch.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.FORWARD, // solution set
    updatingMatch.getInput2().getShipStrategy());
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // workset
    neighborsJoin.getInput1().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // edges
    neighborsJoin.getInput2().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.COMBININGSORT, minIdReducer.getInput().getLocalStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getLocalStrategyKeys());
    Assert.assertEquals(LocalStrategy.NONE, minIdCombiner.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // min id
    updatingMatch.getInput1().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, // solution set
    updatingMatch.getInput2().getLocalStrategy());
    // check the dams
    Assert.assertEquals(TempMode.NONE, iter.getInitialWorksetInput().getTempMode());
    Assert.assertEquals(TempMode.NONE, iter.getInitialSolutionSetInput().getTempMode());
    Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialWorksetInput().getDataExchangeMode());
    Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialSolutionSetInput().getDataExchangeMode());
    JobGraphGenerator jgg = new JobGraphGenerator();
    jgg.compileJobGraph(optPlan);
}

Example 75 with SinkPlanNode

use of org.apache.flink.optimizer.plan.SinkPlanNode in project flink by apache.

the class PartitionOperatorTest method testRangePartitionOperatorPreservesFields.

@Test
public void testRangePartitionOperatorPreservesFields() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L)));
        data.partitionByRange(1).groupBy(1).reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode partitionNode = (SingleInputPlanNode) reducer.getInput().getSource();
        SingleInputPlanNode partitionIDRemover = (SingleInputPlanNode) partitionNode.getInput().getSource();
        assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, partitionNode.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitionIDRemover.getInput().getShipStrategy());
        SourcePlanNode sourcePlanNode = op.getDataSources().iterator().next();
        List<Channel> sourceOutgoingChannels = sourcePlanNode.getOutgoingChannels();
        assertEquals(2, sourceOutgoingChannels.size());
        assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(0).getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(1).getShipStrategy());
        assertEquals(DataExchangeMode.PIPELINED, sourceOutgoingChannels.get(0).getDataExchangeMode());
        assertEquals(DataExchangeMode.BATCH, sourceOutgoingChannels.get(1).getDataExchangeMode());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}