Examples with InvalidProgramException org.apache.flink.api.common.InvalidProgramException used on opensource projects

Example 36 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class CollectionExecutor method executeDataSink.

// --------------------------------------------------------------------------------------------
//  Operator class specific execution methods
// --------------------------------------------------------------------------------------------
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
    Operator<?> inputOp = sink.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
    }
    @SuppressWarnings("unchecked") List<IN> input = (List<IN>) execute(inputOp);
    @SuppressWarnings("unchecked") GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    MetricGroup metrics = new UnregisteredMetricsGroup();
    if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
        ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
    } else {
        ctx = null;
    }
    typedSink.executeOnCollections(input, ctx, executionConfig);
}

Example 37 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class This0AccessFinder method clean.

/**
	 * Tries to clean the closure of the given object, if the object is a non-static inner
	 * class.
	 * 
	 * @param func The object whose closure should be cleaned.
	 * @param checkSerializable Flag to indicate whether serializability should be checked after
	 *                          the closure cleaning attempt.
	 * 
	 * @throws InvalidProgramException Thrown, if 'checkSerializable' is true, and the object was
	 *                                 not serializable after the closure cleaning.
	 * 
	 * @throws RuntimeException A RuntimeException may be thrown, if the code of the class could not
	 *                          be loaded, in order to process during teh closure cleaning.
	 */
public static void clean(Object func, boolean checkSerializable) {
    if (func == null) {
        return;
    }
    final Class<?> cls = func.getClass();
    // First find the field name of the "this$0" field, this can
    // be "this$x" depending on the nesting
    boolean closureAccessed = false;
    for (Field f : cls.getDeclaredFields()) {
        if (f.getName().startsWith("this$")) {
            // found a closure referencing field - now try to clean
            closureAccessed |= cleanThis0(func, cls, f.getName());
        }
    }
    if (checkSerializable) {
        try {
            InstantiationUtil.serializeObject(func);
        } catch (Exception e) {
            String functionType = getSuperClassOrInterfaceName(func.getClass());
            String msg = functionType == null ? (func + " is not serializable.") : ("The implementation of the " + functionType + " is not serializable.");
            if (closureAccessed) {
                msg += " The implementation accesses fields of its enclosing class, which is " + "a common reason for non-serializability. " + "A common solution is to make the function a proper (non-inner) class, or" + "a static inner class.";
            } else {
                msg += " The object probably contains or references non serializable fields.";
            }
            throw new InvalidProgramException(msg, e);
        }
    }
}

Example 38 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class JoinITCase method testDefaultJoinOnTwoCustomTypeInputsWithInnerClassKeyExtractorsDisabledClosureCleaner.

@Test
public void testDefaultJoinOnTwoCustomTypeInputsWithInnerClassKeyExtractorsDisabledClosureCleaner() throws Exception {
    /*
		 * (Default) Join on two custom type inputs with key extractors, check if disableing closure cleaning works
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableClosureCleaner();
    DataSet<CustomType> ds1 = CollectionDataSets.getCustomTypeDataSet(env);
    DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
    boolean correctExceptionTriggered = false;
    try {
        DataSet<Tuple2<CustomType, CustomType>> joinDs = ds1.join(ds2).where(new KeySelector<CustomType, Integer>() {

            @Override
            public Integer getKey(CustomType value) {
                return value.myInt;
            }
        }).equalTo(new KeySelector<CustomType, Integer>() {

            @Override
            public Integer getKey(CustomType value) throws Exception {
                return value.myInt;
            }
        });
    } catch (InvalidProgramException ex) {
        correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException);
    }
    Assert.assertTrue(correctExceptionTriggered);
}

Example 39 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class CoGroupITCase method testCoGroupWithMultipleKeyFieldsWithInnerClassKeyExtractorWithoutClosureCleaner.

@Test
public void testCoGroupWithMultipleKeyFieldsWithInnerClassKeyExtractorWithoutClosureCleaner() throws Exception {
    /*
		 * CoGroup with multiple key fields, test that disabling closure cleaner leads to an exception when using inner
		 * classes.
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableClosureCleaner();
    DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds1 = CollectionDataSets.get5TupleDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
    boolean correctExceptionTriggered = false;
    try {
        DataSet<Tuple3<Integer, Long, String>> coGrouped = ds1.coGroup(ds2).where(new KeySelector<Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, Long>>() {

            @Override
            public Tuple2<Integer, Long> getKey(Tuple5<Integer, Long, Integer, String, Long> t) throws Exception {
                return new Tuple2<Integer, Long>(t.f0, t.f4);
            }
        }).equalTo(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() {

            @Override
            public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> t) {
                return new Tuple2<Integer, Long>(t.f0, t.f1);
            }
        }).with(new CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>() {

            @Override
            public void coGroup(Iterable<Tuple5<Integer, Long, Integer, String, Long>> first, Iterable<Tuple3<Integer, Long, String>> second, Collector<Tuple3<Integer, Long, String>> out) {
                List<String> strs = new ArrayList<String>();
                for (Tuple5<Integer, Long, Integer, String, Long> t : first) {
                    strs.add(t.f3);
                }
                for (Tuple3<Integer, Long, String> t : second) {
                    for (String s : strs) {
                        out.collect(new Tuple3<Integer, Long, String>(t.f0, t.f1, s));
                    }
                }
            }
        });
    } catch (InvalidProgramException ex) {
        correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException);
    }
    Assert.assertTrue(correctExceptionTriggered);
}

Example 40 with InvalidProgramException

use of org.apache.flink.api.common.InvalidProgramException in project flink by apache.

the class OperatorTranslation method translate.

private <T> Operator<T> translate(DataSet<T> dataSet) {
    while (dataSet instanceof NoOpOperator) {
        dataSet = ((NoOpOperator<T>) dataSet).getInput();
    }
    // check if we have already translated that data set (operation or source)
    Operator<?> previous = this.translated.get(dataSet);
    if (previous != null) {
        // The optimizer will merge subsequent binary unions into one n-ary union.
        if (!(dataSet instanceof UnionOperator)) {
            // all other operators are reused.
            @SuppressWarnings("unchecked") Operator<T> typedPrevious = (Operator<T>) previous;
            return typedPrevious;
        }
    }
    Operator<T> dataFlowOp;
    if (dataSet instanceof DataSource) {
        DataSource<T> dataSource = (DataSource<T>) dataSet;
        dataFlowOp = dataSource.translateToDataFlow();
        dataFlowOp.setResources(dataSource.getMinResources(), dataSource.getPreferredResources());
    } else if (dataSet instanceof SingleInputOperator) {
        SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) dataSet;
        dataFlowOp = translateSingleInputOperator(singleInputOperator);
        dataFlowOp.setResources(singleInputOperator.getMinResources(), singleInputOperator.getPreferredResources());
    } else if (dataSet instanceof TwoInputOperator) {
        TwoInputOperator<?, ?, ?, ?> twoInputOperator = (TwoInputOperator<?, ?, ?, ?>) dataSet;
        dataFlowOp = translateTwoInputOperator(twoInputOperator);
        dataFlowOp.setResources(twoInputOperator.getMinResources(), twoInputOperator.getPreferredResources());
    } else if (dataSet instanceof BulkIterationResultSet) {
        BulkIterationResultSet<?> bulkIterationResultSet = (BulkIterationResultSet<?>) dataSet;
        dataFlowOp = translateBulkIteration(bulkIterationResultSet);
        dataFlowOp.setResources(bulkIterationResultSet.getIterationHead().getMinResources(), bulkIterationResultSet.getIterationHead().getPreferredResources());
    } else if (dataSet instanceof DeltaIterationResultSet) {
        DeltaIterationResultSet<?, ?> deltaIterationResultSet = (DeltaIterationResultSet<?, ?>) dataSet;
        dataFlowOp = translateDeltaIteration(deltaIterationResultSet);
        dataFlowOp.setResources(deltaIterationResultSet.getIterationHead().getMinResources(), deltaIterationResultSet.getIterationHead().getPreferredResources());
    } else if (dataSet instanceof DeltaIteration.SolutionSetPlaceHolder || dataSet instanceof DeltaIteration.WorksetPlaceHolder) {
        throw new InvalidProgramException("A data set that is part of a delta iteration was used as a sink or action." + " Did you forget to close the iteration?");
    } else {
        throw new RuntimeException("Error while creating the data flow plan for the program: Unknown operator or data set type: " + dataSet);
    }
    this.translated.put(dataSet, dataFlowOp);
    // take care of broadcast variables
    translateBcVariables(dataSet, dataFlowOp);
    return dataFlowOp;
}