Examples with OutputFormat org.apache.flink.api.common.io.OutputFormat used on opensource projects

Example 1 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class OutputFormatVertex method initializeOnMaster.

@Override
public void initializeOnMaster(ClassLoader loader) throws Exception {
    final TaskConfig cfg = new TaskConfig(getConfiguration());
    UserCodeWrapper<OutputFormat<?>> wrapper;
    try {
        wrapper = cfg.<OutputFormat<?>>getStubWrapper(loader);
    } catch (Throwable t) {
        throw new Exception("Deserializing the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    if (wrapper == null) {
        throw new Exception("No input format present in InputFormatVertex's task configuration.");
    }
    OutputFormat<?> outputFormat;
    try {
        outputFormat = wrapper.getUserCodeObject(OutputFormat.class, loader);
    } catch (Throwable t) {
        throw new Exception("Instantiating the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    try {
        outputFormat.configure(cfg.getStubParameters());
    } catch (Throwable t) {
        throw new Exception("Configuring the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    if (outputFormat instanceof InitializeOnMaster) {
        ((InitializeOnMaster) outputFormat).initializeGlobal(getParallelism());
    }
}

Example 2 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class JobTaskVertexTest method testOutputFormatVertex.

@Test
public void testOutputFormatVertex() {
    try {
        final TestingOutputFormat outputFormat = new TestingOutputFormat();
        final OutputFormatVertex of = new OutputFormatVertex("Name");
        new TaskConfig(of.getConfiguration()).setStubWrapper(new UserCodeObjectWrapper<OutputFormat<?>>(outputFormat));
        final ClassLoader cl = getClass().getClassLoader();
        try {
            of.initializeOnMaster(cl);
            fail("Did not throw expected exception.");
        } catch (TestException e) {
        // all good
        }
        OutputFormatVertex copy = SerializationUtils.clone(of);
        try {
            copy.initializeOnMaster(cl);
            fail("Did not throw expected exception.");
        } catch (TestException e) {
        // all good
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class DataSinkTask method initOutputFormat.

/**
 * Initializes the OutputFormat implementation and configuration.
 *
 * @throws RuntimeException Throws if instance of OutputFormat implementation can not be
 *     obtained.
 */
private void initOutputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);
    final Pair<OperatorID, OutputFormat<IT>> operatorIDAndOutputFormat;
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(config, userCodeClassLoader);
    try {
        operatorIDAndOutputFormat = formatContainer.getUniqueOutputFormat();
        this.format = operatorIDAndOutputFormat.getValue();
        // check if the class is a subclass, if the check is required
        if (!OutputFormat.class.isAssignableFrom(this.format.getClass())) {
            throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" + OutputFormat.class.getName() + "' as is required.");
        }
    } catch (ClassCastException ccex) {
        throw new RuntimeException("The stub class is not a proper subclass of " + OutputFormat.class.getName(), ccex);
    }
    Thread thread = Thread.currentThread();
    ClassLoader original = thread.getContextClassLoader();
    // user code
    try {
        thread.setContextClassLoader(userCodeClassLoader);
        this.format.configure(formatContainer.getParameters(operatorIDAndOutputFormat.getKey()));
    } catch (Throwable t) {
        throw new RuntimeException("The user defined 'configure()' method in the Output Format caused an error: " + t.getMessage(), t);
    } finally {
        thread.setContextClassLoader(original);
    }
}

Example 4 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class StreamingJobGraphGeneratorTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> source = env.addSource(new InputFormatSourceFunction<>(new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)), TypeInformation.of(Long.class)), TypeInformation.of(Long.class)).name("source");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");
    StreamGraph streamGraph = env.getStreamGraph();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    assertEquals(1, jobGraph.getNumberOfVertices());
    JobVertex jobVertex = jobGraph.getVertices().iterator().next();
    assertTrue(jobVertex instanceof InputOutputFormatVertex);
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
    StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
    nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());
    Map<Integer, StreamConfig> chainedConfigs = headConfig.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
    for (StreamConfig config : chainedConfigs.values()) {
        nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
    }
    InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
    assertTrue(sourceFormat instanceof TypeSerializerInputFormat);
    OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
    assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);
    OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
    assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}

Example 5 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class OutputFormatVertex method finalizeOnMaster.

@Override
public void finalizeOnMaster(ClassLoader loader) throws Exception {
    final TaskConfig cfg = new TaskConfig(getConfiguration());
    UserCodeWrapper<OutputFormat<?>> wrapper;
    try {
        wrapper = cfg.<OutputFormat<?>>getStubWrapper(loader);
    } catch (Throwable t) {
        throw new Exception("Deserializing the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    if (wrapper == null) {
        throw new Exception("No input format present in InputFormatVertex's task configuration.");
    }
    OutputFormat<?> outputFormat;
    try {
        outputFormat = wrapper.getUserCodeObject(OutputFormat.class, loader);
    } catch (Throwable t) {
        throw new Exception("Instantiating the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    try {
        outputFormat.configure(cfg.getStubParameters());
    } catch (Throwable t) {
        throw new Exception("Configuring the OutputFormat (" + formatDescription + ") failed: " + t.getMessage(), t);
    }
    if (outputFormat instanceof FinalizeOnMaster) {
        ((FinalizeOnMaster) outputFormat).finalizeGlobal(getParallelism());
    }
}