Examples with CompilerException org.apache.flink.optimizer.CompilerException used on opensource projects

Example 16 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class JavaApiPostPass method traverse.

protected void traverse(PlanNode node) {
    if (!alreadyDone.add(node)) {
        // already worked on that one
        return;
    }
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
        // descend to the input channel
        SinkPlanNode sn = (SinkPlanNode) node;
        Channel inchannel = sn.getInput();
        traverseChannel(inchannel);
    } else if (node instanceof SourcePlanNode) {
        TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
        ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
    } else if (node instanceof BulkIterationPlanNode) {
        BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
        if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
        }
        // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
        if (iterationNode.getRootOfTerminationCriterion() != null) {
            SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
            traverseChannel(addMapper.getInput());
        }
        BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
        // set the serializer
        iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
        // done, we can now propagate our info down
        traverseChannel(iterationNode.getInput());
        traverse(iterationNode.getRootOfStepFunction());
    } else if (node instanceof WorksetIterationPlanNode) {
        WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
        if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
        }
        if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
            throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
        }
        DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
        // set the serializers and comparators for the workset iteration
        iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
        iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
        iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
        // traverse the inputs
        traverseChannel(iterationNode.getInput1());
        traverseChannel(iterationNode.getInput2());
        // traverse the step function
        traverse(iterationNode.getSolutionSetDeltaPlanNode());
        traverse(iterationNode.getNextWorkSetPlanNode());
    } else if (node instanceof SingleInputPlanNode) {
        SingleInputPlanNode sn = (SingleInputPlanNode) node;
        if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
            // Special case for delta iterations
            if (sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
                traverseChannel(sn.getInput());
                return;
            } else {
                throw new RuntimeException("Wrong operator type found in post pass.");
            }
        }
        SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        for (int i = 0; i < sn.getDriverStrategy().getNumRequiredComparators(); i++) {
            sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
        }
        // done, we can now propagate our info down
        traverseChannel(sn.getInput());
        // don't forget the broadcast inputs
        for (Channel c : sn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else if (node instanceof DualInputPlanNode) {
        DualInputPlanNode dn = (DualInputPlanNode) node;
        if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
            throw new RuntimeException("Wrong operator type found in post pass.");
        }
        DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
        // parameterize the node's driver strategy
        if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
            dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(), getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
            dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(), getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));
            dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dualInputOperator.getOperatorInfo().getSecondInputType()));
        }
        traverseChannel(dn.getInput1());
        traverseChannel(dn.getInput2());
        // don't forget the broadcast inputs
        for (Channel c : dn.getBroadcastInputs()) {
            traverseChannel(c);
        }
    } else // catch the sources of the iterative step functions
    if (node instanceof BulkPartialSolutionPlanNode || node instanceof SolutionSetPlanNode || node instanceof WorksetPlanNode) {
    // Do nothing :D
    } else if (node instanceof NAryUnionPlanNode) {
        // Traverse to all child channels
        for (Channel channel : node.getInputs()) {
            traverseChannel(channel);
        }
    } else {
        throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
    }
}

Example 17 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class PlanFinalizer method preVisit.

@Override
public boolean preVisit(PlanNode visitable) {
    // if we come here again, prevent a further descend
    if (!this.allNodes.add(visitable)) {
        return false;
    }
    if (visitable instanceof SinkPlanNode) {
        this.sinks.add((SinkPlanNode) visitable);
    } else if (visitable instanceof SourcePlanNode) {
        this.sources.add((SourcePlanNode) visitable);
    } else if (visitable instanceof BinaryUnionPlanNode) {
        BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        if (unionNode.unionsStaticAndDynamicPath()) {
            unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
        }
    } else if (visitable instanceof BulkPartialSolutionPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
    } else if (visitable instanceof WorksetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    } else if (visitable instanceof SolutionSetPlanNode) {
        // tell the partial solution about the iteration node that contains it
        final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
        final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
        // sanity check!
        if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
            throw new CompilerException("Bug: Error finalizing the plan. " + "Cannot associate the node for a partial solutions with its containing iteration.");
        }
        sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
    }
    // one child candidate could have been referenced by multiple parents.
    for (Channel conn : visitable.getInputs()) {
        conn.setTarget(visitable);
        conn.getSource().addOutgoingChannel(conn);
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        c.setTarget(visitable);
        c.getSource().addOutgoingChannel(c);
    }
    // count the memory consumption
    this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
    for (Channel c : visitable.getInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    for (Channel c : visitable.getBroadcastInputs()) {
        if (c.getLocalStrategy().dams()) {
            this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
            this.memoryConsumerWeights++;
        }
    }
    // pass the visitor to the iteraton's step function
    if (visitable instanceof IterationPlanNode) {
        // push the iteration node onto the stack
        final IterationPlanNode iterNode = (IterationPlanNode) visitable;
        this.stackOfIterationNodes.addLast(iterNode);
        // recurse
        ((IterationPlanNode) visitable).acceptForStepFunction(this);
        // pop the iteration node from the stack
        this.stackOfIterationNodes.removeLast();
    }
    return true;
}

Example 18 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class PlanNode method setBroadcastInputs.

/**
	 * Sets a list of all broadcast inputs attached to this node.
	 */
public void setBroadcastInputs(List<NamedChannel> broadcastInputs) {
    if (broadcastInputs != null) {
        this.broadcastInputs = broadcastInputs;
        // update the branch map
        for (NamedChannel nc : broadcastInputs) {
            PlanNode source = nc.getSource();
            mergeBranchPlanMaps(branchPlan, source.branchPlan);
        }
    }
    // do a sanity check that if we are branching, we have now candidates for each branch point
    if (this.template.hasUnclosedBranches()) {
        if (this.branchPlan == null) {
            throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
        }
        for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
            OptimizerNode brancher = uc.getBranchingNode();
            if (this.branchPlan.get(brancher) == null) {
                throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
            }
        }
    }
}

Example 19 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class WorksetIterationPlanNode method mergeBranchPlanMaps.

protected void mergeBranchPlanMaps() {
    Map<OptimizerNode, PlanNode> branchPlan1 = input1.getSource().branchPlan;
    Map<OptimizerNode, PlanNode> branchPlan2 = input2.getSource().branchPlan;
    // merge the branchPlan maps according the template's uncloseBranchesStack
    if (this.template.hasUnclosedBranches()) {
        if (this.branchPlan == null) {
            this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
        }
        for (OptimizerNode.UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
            OptimizerNode brancher = uc.getBranchingNode();
            PlanNode selectedCandidate = null;
            if (branchPlan1 != null) {
                // predecessor 1 has branching children, see if it got the branch we are looking for
                selectedCandidate = branchPlan1.get(brancher);
            }
            if (selectedCandidate == null && branchPlan2 != null) {
                // predecessor 2 has branching children, see if it got the branch we are looking for
                selectedCandidate = branchPlan2.get(brancher);
            }
            if (selectedCandidate == null && getSolutionSetDeltaPlanNode() != null && getSolutionSetDeltaPlanNode().branchPlan != null) {
                selectedCandidate = getSolutionSetDeltaPlanNode().branchPlan.get(brancher);
            }
            if (selectedCandidate == null && getNextWorkSetPlanNode() != null && getNextWorkSetPlanNode().branchPlan != null) {
                selectedCandidate = getNextWorkSetPlanNode().branchPlan.get(brancher);
            }
            if (selectedCandidate == null) {
                throw new CompilerException("Candidates for a node with open branches are missing information about the selected candidate ");
            }
            this.branchPlan.put(brancher, selectedCandidate);
        }
    }
}

Example 20 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class RequestedGlobalProperties method parameterizeChannel.

/**
	 * Parametrizes the ship strategy fields of a channel such that the channel produces
	 * the desired global properties.
	 * 
	 * @param channel The channel to parametrize.
	 * @param globalDopChange Flag indicating whether the parallelism changes
	 *                        between sender and receiver.
	 * @param exchangeMode The mode of data exchange (pipelined, always batch,
	 *                     batch only on shuffle, ...)
	 * @param breakPipeline Indicates whether this data exchange should break
	 *                      pipelines (unless pipelines are forced).
	 */
public void parameterizeChannel(Channel channel, boolean globalDopChange, ExecutionMode exchangeMode, boolean breakPipeline) {
    // safety check. Fully replicated input must be preserved.
    if (channel.getSource().getGlobalProperties().isFullyReplicated() && !(this.partitioning == PartitioningProperty.FULL_REPLICATION || this.partitioning == PartitioningProperty.ANY_DISTRIBUTION)) {
        throw new CompilerException("Fully replicated input must be preserved " + "and may not be converted into another global property.");
    }
    // the same, randomly repartition otherwise
    if (isTrivial() || this.partitioning == PartitioningProperty.ANY_DISTRIBUTION) {
        ShipStrategyType shipStrategy = globalDopChange ? ShipStrategyType.PARTITION_RANDOM : ShipStrategyType.FORWARD;
        DataExchangeMode em = DataExchangeMode.select(exchangeMode, shipStrategy, breakPipeline);
        channel.setShipStrategy(shipStrategy, em);
        return;
    }
    final GlobalProperties inGlobals = channel.getSource().getGlobalProperties();
    // if we have no global parallelism change, check if we have already compatible global properties
    if (!globalDopChange && isMetBy(inGlobals)) {
        DataExchangeMode em = DataExchangeMode.select(exchangeMode, ShipStrategyType.FORWARD, breakPipeline);
        channel.setShipStrategy(ShipStrategyType.FORWARD, em);
        return;
    }
    // if we fall through the conditions until here, we need to re-establish
    ShipStrategyType shipType;
    FieldList partitionKeys;
    boolean[] sortDirection;
    Partitioner<?> partitioner;
    switch(this.partitioning) {
        case FULL_REPLICATION:
            shipType = ShipStrategyType.BROADCAST;
            partitionKeys = null;
            sortDirection = null;
            partitioner = null;
            break;
        case ANY_PARTITIONING:
        case HASH_PARTITIONED:
            shipType = ShipStrategyType.PARTITION_HASH;
            partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
            sortDirection = null;
            partitioner = null;
            break;
        case RANGE_PARTITIONED:
            shipType = ShipStrategyType.PARTITION_RANGE;
            partitionKeys = this.ordering.getInvolvedIndexes();
            sortDirection = this.ordering.getFieldSortDirections();
            partitioner = null;
            if (this.dataDistribution != null) {
                channel.setDataDistribution(this.dataDistribution);
            }
            break;
        case FORCED_REBALANCED:
            shipType = ShipStrategyType.PARTITION_FORCED_REBALANCE;
            partitionKeys = null;
            sortDirection = null;
            partitioner = null;
            break;
        case CUSTOM_PARTITIONING:
            shipType = ShipStrategyType.PARTITION_CUSTOM;
            partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
            sortDirection = null;
            partitioner = this.customPartitioner;
            break;
        default:
            throw new CompilerException("Invalid partitioning to create through a data exchange: " + this.partitioning.name());
    }
    DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
    channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
}