Examples with InputFormat org.apache.flink.api.common.io.InputFormat used on opensource projects

Example 1 with InputFormat

use of org.apache.flink.api.common.io.InputFormat in project flink by apache.

the class JobTaskVertexTest method testInputFormatVertex.

@Test
public void testInputFormatVertex() {
    try {
        final TestInputFormat inputFormat = new TestInputFormat();
        final InputFormatVertex vertex = new InputFormatVertex("Name");
        new TaskConfig(vertex.getConfiguration()).setStubWrapper(new UserCodeObjectWrapper<InputFormat<?, ?>>(inputFormat));
        final ClassLoader cl = getClass().getClassLoader();
        vertex.initializeOnMaster(cl);
        InputSplit[] splits = vertex.getInputSplitSource().createInputSplits(77);
        assertNotNull(splits);
        assertEquals(1, splits.length);
        assertEquals(TestSplit.class, splits[0].getClass());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with InputFormat

use of org.apache.flink.api.common.io.InputFormat in project flink by apache.

the class GenericDataSourceBase method executeOnCollections.

// --------------------------------------------------------------------------------------------
protected List<OUT> executeOnCollections(RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
    @SuppressWarnings("unchecked") InputFormat<OUT, InputSplit> inputFormat = (InputFormat<OUT, InputSplit>) this.formatWrapper.getUserCodeObject();
    //configure the input format
    inputFormat.configure(this.parameters);
    //open the input format
    if (inputFormat instanceof RichInputFormat) {
        ((RichInputFormat) inputFormat).setRuntimeContext(ctx);
        ((RichInputFormat) inputFormat).openInputFormat();
    }
    List<OUT> result = new ArrayList<OUT>();
    // splits
    InputSplit[] splits = inputFormat.createInputSplits(1);
    TypeSerializer<OUT> serializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);
    for (InputSplit split : splits) {
        inputFormat.open(split);
        while (!inputFormat.reachedEnd()) {
            OUT next = inputFormat.nextRecord(serializer.createInstance());
            if (next != null) {
                result.add(serializer.copy(next));
            }
        }
        inputFormat.close();
    }
    //close the input format
    if (inputFormat instanceof RichInputFormat) {
        ((RichInputFormat) inputFormat).closeInputFormat();
    }
    return result;
}

Example 3 with InputFormat

use of org.apache.flink.api.common.io.InputFormat in project flink by apache.

the class InputFormatVertex method initializeOnMaster.

@Override
public void initializeOnMaster(ClassLoader loader) throws Exception {
    final TaskConfig cfg = new TaskConfig(getConfiguration());
    // deserialize from the payload
    UserCodeWrapper<InputFormat<?, ?>> wrapper;
    try {
        wrapper = cfg.getStubWrapper(loader);
    } catch (Throwable t) {
        throw new Exception("Deserializing the InputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    if (wrapper == null) {
        throw new Exception("No input format present in InputFormatVertex's task configuration.");
    }
    // instantiate, if necessary
    InputFormat<?, ?> inputFormat;
    try {
        inputFormat = wrapper.getUserCodeObject(InputFormat.class, loader);
    } catch (Throwable t) {
        throw new Exception("Instantiating the InputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    Thread thread = Thread.currentThread();
    ClassLoader original = thread.getContextClassLoader();
    // configure
    try {
        thread.setContextClassLoader(loader);
        inputFormat.configure(cfg.getStubParameters());
    } catch (Throwable t) {
        throw new Exception("Configuring the InputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    } finally {
        thread.setContextClassLoader(original);
    }
    setInputSplitSource(inputFormat);
}

Example 4 with InputFormat

use of org.apache.flink.api.common.io.InputFormat in project flink by apache.

the class DataSourceTask method initInputFormat.

/**
	 * Initializes the InputFormat implementation and configuration.
	 * 
	 * @throws RuntimeException
	 *         Throws if instance of InputFormat implementation can not be
	 *         obtained.
	 */
private void initInputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);
    try {
        this.format = config.<InputFormat<OT, InputSplit>>getStubWrapper(userCodeClassLoader).getUserCodeObject(InputFormat.class, userCodeClassLoader);
        // check if the class is a subclass, if the check is required
        if (!InputFormat.class.isAssignableFrom(this.format.getClass())) {
            throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" + InputFormat.class.getName() + "' as is required.");
        }
    } catch (ClassCastException ccex) {
        throw new RuntimeException("The stub class is not a proper subclass of " + InputFormat.class.getName(), ccex);
    }
    Thread thread = Thread.currentThread();
    ClassLoader original = thread.getContextClassLoader();
    // configure the stub. catch exceptions here extra, to report them as originating from the user code
    try {
        thread.setContextClassLoader(userCodeClassLoader);
        this.format.configure(this.config.getStubParameters());
    } catch (Throwable t) {
        throw new RuntimeException("The user defined 'configure()' method caused an error: " + t.getMessage(), t);
    } finally {
        thread.setContextClassLoader(original);
    }
    // get the factory for the type serializer
    this.serializerFactory = this.config.getOutputSerializer(userCodeClassLoader);
}