Examples with SourcePlanNode org.apache.flink.optimizer.plan.SourcePlanNode used on opensource projects

Example 11 with SourcePlanNode

use of org.apache.flink.optimizer.plan.SourcePlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testNoPartialSolutionFoundSingleInputOnly.

@Test
public void testNoPartialSolutionFoundSingleInputOnly() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = new LocalProperties();
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
            assertTrue(report == NO_PARTIAL_SOLUTION);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with SourcePlanNode

use of org.apache.flink.optimizer.plan.SourcePlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testSingleInputOperatorsChainOfThree.

@Test
public void testSingleInputOperatorsChainOfThree() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toMap3 = new Channel(map2);
        SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP);
        // set local strategy in first channel, so later non matching local properties do not
        // matter
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap3.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            toMap3.setRequiredGlobalProps(null);
            toMap3.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // set global strategy in first channel, so later non matching global properties do not
        // matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(5, 3));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 3));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(1, 2), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap3.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            toMap3.setRequiredGlobalProps(reqGp);
            toMap3.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 13 with SourcePlanNode

use of org.apache.flink.optimizer.plan.SourcePlanNode in project flink by apache.

the class FeedbackPropertiesMatchTest method testTwoOperatorsBothDependent.

@Test
public void testTwoOperatorsBothDependent() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
        Channel toMap1 = new Channel(target);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(target);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toJoin1 = new Channel(map1);
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        Channel toJoin2 = new Channel(map2);
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
        Channel toAfterJoin = new Channel(join);
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
        // no properties from the partial solution, no required properties
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.EMPTY;
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some properties from the partial solution, no required properties
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on one input and met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on both input and met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setRequiredGlobalProps(rgp);
            toJoin1.setRequiredLocalProps(rlp);
            toJoin2.setRequiredGlobalProps(rgp);
            toJoin2.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // test requirements on both inputs, one not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
            rgp1.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp1 = new RequestedLocalProperties();
            rlp1.setGroupedFields(new FieldList(2, 1));
            RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
            rgp2.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp2 = new RequestedLocalProperties();
            rlp2.setGroupedFields(new FieldList(0, 3));
            toJoin1.setRequiredGlobalProps(rgp1);
            toJoin1.setRequiredLocalProps(rlp1);
            toJoin2.setRequiredGlobalProps(rgp2);
            toJoin2.setRequiredLocalProps(rlp2);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // test override on both inputs, later requirement ignored
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(1));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(0, 3));
            toJoin1.setRequiredGlobalProps(null);
            toJoin1.setRequiredLocalProps(null);
            toJoin2.setRequiredGlobalProps(null);
            toJoin2.setRequiredLocalProps(null);
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // test override on one inputs, later requirement met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(0));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(2, 1));
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(PENDING, report);
        }
        // test override on one input, later requirement not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(3));
            RequestedLocalProperties rlp = new RequestedLocalProperties();
            rlp.setGroupedFields(new FieldList(77, 69));
            toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(rlp);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // test override on one input locally, later global requirement not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(0));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties rgp = new RequestedGlobalProperties();
            rgp.setHashPartitioned(new FieldList(3));
            toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(3), new boolean[] { false });
            toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toJoin1.setLocalStrategy(LocalStrategy.NONE);
            toAfterJoin.setRequiredGlobalProps(rgp);
            toAfterJoin.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with SourcePlanNode

use of org.apache.flink.optimizer.plan.SourcePlanNode in project flink by apache.

the class JavaApiPostPass method traverse.

protected void traverse(PlanNode node) {
    if (!alreadyDone.add(node)) {
        // already worked on that one
        return;
    }
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
        // descend to the input channel
        SinkPlanNode sn = (SinkPlanNode) node;
        Channel inchannel = sn.getInput();
        traverseChannel(inchannel);
    } else if (node instanceof SourcePlanNode) {
        TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
        ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
    } else if (node instanceof BulkIterationPlanNode) {
        BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
        if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
        }
        // utilities. Needed in case of intermediate termination criterion
        if (iterationNode.getRootOfTerminationCriterion() != null) {
            SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
            traverseChannel(addMapper.getInput());
        }
        BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
        // set the serializer
        iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
        // done, we can now propagate our info down
        traverseChannel(iterationNode.getInput());
        traverse(iterationNode.getRootOfStepFunction());
    } else if (node instanceof WorksetIterationPlanNode) {
        WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
        if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
        }
        if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
        }
        DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
        // set the serializers and comparators for the workset iteration
        iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
        iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
        iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
        // traverse the inputs
        traverseChannel(iterationNode.getInput1());
        traverseChannel(iterationNode.getInput2());
        // traverse the step function
        traverse(iterationNode.getSolutionSetDeltaPlanNode());
        traverse(iterationNode.getNextWorkSetPlanNode());
    } else if (node instanceof SingleInputPlanNode) {
        SingleInputPlanNode sn = (SingleInputPlanNode) node;
        if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
            // Special case for delta iterations
            if (sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
                traverseChannel(sn.getInput());
                return;
            } else {
                throw new RuntimeException("Wrong operator type found in post pass.");
            }
        }
        SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        for (int i = 0; i < sn.getDriverStrategy().getNumRequiredComparators(); i++) {
            sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
        }
        // done, we can now propagate our info down
        traverseChannel(sn.getInput());
        // don't forget the broadcast inputs
        for (Channel c : sn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else if (node instanceof DualInputPlanNode) {
        DualInputPlanNode dn = (DualInputPlanNode) node;
        if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
            throw new RuntimeException("Wrong operator type found in post pass.");
        }
        DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
            dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(), getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
            dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(), getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));
            dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dualInputOperator.getOperatorInfo().getSecondInputType()));
        }
        traverseChannel(dn.getInput1());
        traverseChannel(dn.getInput2());
        // don't forget the broadcast inputs
        for (Channel c : dn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else // catch the sources of the iterative step functions
    if (node instanceof BulkPartialSolutionPlanNode || node instanceof SolutionSetPlanNode || node instanceof WorksetPlanNode) {
    // Do nothing :D
    } else if (node instanceof NAryUnionPlanNode) {
        // Traverse to all child channels
        for (Channel channel : node.getInputs()) {
            traverseChannel(channel);
        }
    } else {
        throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
    }
}

Example 15 with SourcePlanNode

use of org.apache.flink.optimizer.plan.SourcePlanNode in project flink by apache.

the class DataSourceNode method getAlternativePlans.

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    SourcePlanNode candidate = new SourcePlanNode(this, "DataSource (" + this.getOperator().getName() + ")", this.gprops, this.lprops);
    if (!replicatedInput) {
        candidate.updatePropertiesWithUniqueSets(getUniqueFields());
        final Costs costs = new Costs();
        if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) && this.estimatedOutputSize >= 0) {
            estimator.addFileInputCost(this.estimatedOutputSize, costs);
        }
        candidate.setCosts(costs);
    } else {
        // replicated input
        final Costs costs = new Costs();
        InputFormat<?, ?> inputFormat = ((ReplicatingInputFormat<?, ?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
        if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) && this.estimatedOutputSize >= 0) {
            estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs);
        }
        candidate.setCosts(costs);
    }
    // since there is only a single plan for the data-source, return a list with that element
    // only
    List<PlanNode> plans = new ArrayList<PlanNode>(1);
    plans.add(candidate);
    this.cachedPlans = plans;
    return plans;
}