Examples with InvalidProgramException org.apache.flink.api.common.InvalidProgramException used on opensource projects

Example 56 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink-mirror by flink-ci.

the class CoGroupOperator method translateToDataFlow.

@Override
@Internal
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;
    if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroup(selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys2 instanceof SelectorFunctionKeys) {
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroupRight(logicalKeyPositions1, selectorKeys2, function, getInput1Type(), getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys1 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        po = translateSelectorFunctionCoGroupLeft(selectorKeys1, logicalKeyPositions2, function, getInput2Type(), getResultType(), name, input1, input2);
    } else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op = new CoGroupOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        op.setFirstInput(input1);
        op.setSecondInput(input2);
        po = op;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
    // configure shared characteristics
    po.setParallelism(getParallelism());
    po.setCustomPartitioner(customPartitioner);
    if (groupSortKeyOrderFirst.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputOne(o);
    }
    if (groupSortKeyOrderSecond.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputTwo(o);
    }
    return po;
}

Example 57 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink-mirror by flink-ci.

the class LocalEnvironment method validateAndGetConfiguration.

private static Configuration validateAndGetConfiguration(final Configuration configuration) {
    if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) {
        throw new InvalidProgramException("The LocalEnvironment cannot be instantiated when running in a pre-defined context " + "(such as Command Line Client or TestEnvironment)");
    }
    final Configuration effectiveConfiguration = new Configuration(checkNotNull(configuration));
    effectiveConfiguration.set(DeploymentOptions.TARGET, "local");
    effectiveConfiguration.set(DeploymentOptions.ATTACHED, true);
    return effectiveConfiguration;
}

Example 58 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink-mirror by flink-ci.

the class CoGroupRawOperator method translateToDataFlow.

@Override
protected org.apache.flink.api.common.operators.base.CoGroupRawOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupRawOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> po = new CoGroupRawOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        // set inputs
        po.setFirstInput(input1);
        po.setSecondInput(input2);
        // set dop
        po.setParallelism(this.getParallelism());
        return po;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
}

Example 59 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink-mirror by flink-ci.

the class This0AccessFinder method clean.

private static void clean(Object func, ExecutionConfig.ClosureCleanerLevel level, boolean checkSerializable, Set<Object> visited) {
    if (func == null) {
        return;
    }
    if (!visited.add(func)) {
        return;
    }
    final Class<?> cls = func.getClass();
    if (ClassUtils.isPrimitiveOrWrapper(cls)) {
        return;
    }
    if (usesCustomSerialization(cls)) {
        return;
    }
    // First find the field name of the "this$0" field, this can
    // be "this$x" depending on the nesting
    boolean closureAccessed = false;
    for (Field f : cls.getDeclaredFields()) {
        if (f.getName().startsWith("this$")) {
            // found a closure referencing field - now try to clean
            closureAccessed |= cleanThis0(func, cls, f.getName());
        } else {
            Object fieldObject;
            try {
                f.setAccessible(true);
                fieldObject = f.get(func);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(String.format("Can not access to the %s field in Class %s", f.getName(), func.getClass()));
            }
            /*
                 * we should do a deep clean when we encounter an anonymous class, inner class and local class, but should
                 * skip the class with custom serialize method.
                 *
                 * There are five kinds of classes (or interfaces):
                 * a) Top level classes
                 * b) Nested classes (static member classes)
                 * c) Inner classes (non-static member classes)
                 * d) Local classes (named classes declared within a method)
                 * e) Anonymous classes
                 */
            if (level == ExecutionConfig.ClosureCleanerLevel.RECURSIVE && needsRecursion(f, fieldObject)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Dig to clean the {}", fieldObject.getClass().getName());
                }
                clean(fieldObject, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true, visited);
            }
        }
    }
    if (checkSerializable) {
        try {
            InstantiationUtil.serializeObject(func);
        } catch (Exception e) {
            String functionType = getSuperClassOrInterfaceName(func.getClass());
            String msg = functionType == null ? (func + " is not serializable.") : ("The implementation of the " + functionType + " is not serializable.");
            if (closureAccessed) {
                msg += " The implementation accesses fields of its enclosing class, which is " + "a common reason for non-serializability. " + "A common solution is to make the function a proper (non-inner) class, or " + "a static inner class.";
            } else {
                msg += " The object probably contains or references non serializable fields.";
            }
            throw new InvalidProgramException(msg, e);
        }
    }
}

Example 60 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink-mirror by flink-ci.

the class OperatorTranslation method translate.

private <T> Operator<T> translate(DataSet<T> dataSet) {
    while (dataSet instanceof NoOpOperator) {
        dataSet = ((NoOpOperator<T>) dataSet).getInput();
    }
    // check if we have already translated that data set (operation or source)
    Operator<?> previous = this.translated.get(dataSet);
    if (previous != null) {
        // The optimizer will merge subsequent binary unions into one n-ary union.
        if (!(dataSet instanceof UnionOperator)) {
            // all other operators are reused.
            @SuppressWarnings("unchecked") Operator<T> typedPrevious = (Operator<T>) previous;
            return typedPrevious;
        }
    }
    Operator<T> dataFlowOp;
    if (dataSet instanceof DataSource) {
        DataSource<T> dataSource = (DataSource<T>) dataSet;
        dataFlowOp = dataSource.translateToDataFlow();
        dataFlowOp.setResources(dataSource.getMinResources(), dataSource.getPreferredResources());
    } else if (dataSet instanceof SingleInputOperator) {
        SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) dataSet;
        dataFlowOp = translateSingleInputOperator(singleInputOperator);
        dataFlowOp.setResources(singleInputOperator.getMinResources(), singleInputOperator.getPreferredResources());
    } else if (dataSet instanceof TwoInputOperator) {
        TwoInputOperator<?, ?, ?, ?> twoInputOperator = (TwoInputOperator<?, ?, ?, ?>) dataSet;
        dataFlowOp = translateTwoInputOperator(twoInputOperator);
        dataFlowOp.setResources(twoInputOperator.getMinResources(), twoInputOperator.getPreferredResources());
    } else if (dataSet instanceof BulkIterationResultSet) {
        BulkIterationResultSet<?> bulkIterationResultSet = (BulkIterationResultSet<?>) dataSet;
        dataFlowOp = translateBulkIteration(bulkIterationResultSet);
        dataFlowOp.setResources(bulkIterationResultSet.getIterationHead().getMinResources(), bulkIterationResultSet.getIterationHead().getPreferredResources());
    } else if (dataSet instanceof DeltaIterationResultSet) {
        DeltaIterationResultSet<?, ?> deltaIterationResultSet = (DeltaIterationResultSet<?, ?>) dataSet;
        dataFlowOp = translateDeltaIteration(deltaIterationResultSet);
        dataFlowOp.setResources(deltaIterationResultSet.getIterationHead().getMinResources(), deltaIterationResultSet.getIterationHead().getPreferredResources());
    } else if (dataSet instanceof DeltaIteration.SolutionSetPlaceHolder || dataSet instanceof DeltaIteration.WorksetPlaceHolder) {
        throw new InvalidProgramException("A data set that is part of a delta iteration was used as a sink or action." + " Did you forget to close the iteration?");
    } else {
        throw new RuntimeException("Error while creating the data flow plan for the program: Unknown operator or data set type: " + dataSet);
    }
    this.translated.put(dataSet, dataFlowOp);
    // take care of broadcast variables
    translateBcVariables(dataSet, dataFlowOp);
    return dataFlowOp;
}