Examples with InvalidProgramException org.apache.flink.api.common.InvalidProgramException used on opensource projects

Example 11 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class RangePartitionRewriter method postVisit.

@Override
public void postVisit(PlanNode node) {
    if (node instanceof IterationPlanNode) {
        IterationPlanNode iNode = (IterationPlanNode) node;
        if (!visitedIterationNodes.contains(iNode)) {
            visitedIterationNodes.add(iNode);
            iNode.acceptForStepFunction(this);
        }
    }
    final Iterable<Channel> inputChannels = node.getInputs();
    for (Channel channel : inputChannels) {
        ShipStrategyType shipStrategy = channel.getShipStrategy();
        // times.
        if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {
            if (channel.getDataDistribution() == null) {
                if (node.isOnDynamicPath()) {
                    throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
                }
                PlanNode channelSource = channel.getSource();
                List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
                channelSource.getOutgoingChannels().remove(channel);
                channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
            }
        }
    }
}

Example 12 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class CollectionExecutor method executeUnaryOperator.

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep, JobID jobID) throws Exception {
    Operator<?> inputOp = operator.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
    }
    @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep, jobID);
    @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
        ctx = createContext(superStep, taskInfo, jobID);
        for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
            List<?> bcData = execute(bcInputs.getValue(), jobID);
            ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
        }
    } else {
        ctx = null;
    }
    return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}

Example 13 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class SplitDataProperties method splitsOrderedBy.

/**
 * Defines that the data within an input split is sorted on the fields defined by the field
 * expressions in the specified orders. Multiple field expressions must be separated by the
 * semicolon ';' character. All records of an input split must be emitted by the input format in
 * the defined order.
 *
 * <p><b> IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong
 * results! </b>
 *
 * @param orderFields The field expressions of the grouping key.
 * @param orders The orders of the fields.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsOrderedBy(String orderFields, Order[] orders) {
    if (orderFields == null || orders == null) {
        throw new InvalidProgramException("OrderFields or Orders may not be null.");
    }
    String[] orderKeysA = orderFields.split(";");
    if (orderKeysA.length == 0) {
        throw new InvalidProgramException("OrderFields may not be empty.");
    } else if (orders.length == 0) {
        throw new InvalidProgramException("Orders may not be empty");
    } else if (orderKeysA.length != orders.length) {
        throw new InvalidProgramException("Number of OrderFields and Orders must match.");
    }
    if (this.splitGroupKeys != null) {
        throw new InvalidProgramException("DataSource may either be grouped or sorted.");
    }
    this.splitOrdering = new Ordering();
    for (int i = 0; i < orderKeysA.length; i++) {
        String keyExp = orderKeysA[i];
        Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type);
        int[] flatKeys = ek.computeLogicalKeyPositions();
        for (int key : flatKeys) {
            // check for duplicates
            for (int okey : splitOrdering.getFieldPositions()) {
                if (key == okey) {
                    throw new InvalidProgramException("Duplicate field in field expression " + keyExp);
                }
            }
            // append key
            this.splitOrdering.appendOrdering(key, null, orders[i]);
        }
    }
    return this;
}

Example 14 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class SplitDataProperties method splitsOrderedBy.

/**
 * Defines that the data within an input split is sorted on the fields defined by the field
 * positions in the specified orders. All records of an input split must be emitted by the input
 * format in the defined order.
 *
 * <p><b> IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong
 * results! </b>
 *
 * @param orderFields The field positions of the grouping keys.
 * @param orders The orders of the fields.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsOrderedBy(int[] orderFields, Order[] orders) {
    if (orderFields == null || orders == null) {
        throw new InvalidProgramException("OrderFields or Orders may not be null.");
    } else if (orderFields.length == 0) {
        throw new InvalidProgramException("OrderFields may not be empty.");
    } else if (orders.length == 0) {
        throw new InvalidProgramException("Orders may not be empty");
    } else if (orderFields.length != orders.length) {
        throw new InvalidProgramException("Number of OrderFields and Orders must match.");
    }
    if (this.splitGroupKeys != null) {
        throw new InvalidProgramException("DataSource may either be grouped or sorted.");
    }
    this.splitOrdering = new Ordering();
    for (int i = 0; i < orderFields.length; i++) {
        int pos = orderFields[i];
        int[] flatKeys = this.getAllFlatKeys(new int[] { pos });
        for (int key : flatKeys) {
            // check for duplicates
            for (int okey : splitOrdering.getFieldPositions()) {
                if (key == okey) {
                    throw new InvalidProgramException("Duplicate field in the field expression " + pos);
                }
            }
            // append key
            this.splitOrdering.appendOrdering(key, null, orders[i]);
        }
    }
    return this;
}

Example 15 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class CoGroupRawOperator method translateToDataFlow.

@Override
protected org.apache.flink.api.common.operators.base.CoGroupRawOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupRawOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> po = new CoGroupRawOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        // set inputs
        po.setFirstInput(input1);
        po.setSecondInput(input2);
        // set dop
        po.setParallelism(this.getParallelism());
        return po;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
}