Examples with DiscardingOutputFormat org.apache.flink.api.java.io.DiscardingOutputFormat used on opensource projects

Example 46 with DiscardingOutputFormat

use of org.apache.flink.api.java.io.DiscardingOutputFormat in project flink by apache.

the class InputOutputFormatContainerTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID1 = new OperatorID();
    formatContainer.addInputFormat(operatorID1, new TestInputFormat("test input format"));
    formatContainer.addParameters(operatorID1, "parameter1", "abc123");
    OperatorID operatorID2 = new OperatorID();
    formatContainer.addOutputFormat(operatorID2, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID2, "parameter1", "bcd234");
    OperatorID operatorID3 = new OperatorID();
    formatContainer.addOutputFormat(operatorID3, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID3, "parameter1", "cde345");
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = loadedFormatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    // verify the input format
    TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID1).getUserCodeObject();
    assertEquals("test input format", inputFormat.getName());
    Configuration inputFormatParams = loadedFormatContainer.getParameters(operatorID1);
    assertEquals(1, inputFormatParams.keySet().size());
    assertEquals("abc123", inputFormatParams.getString("parameter1", null));
    // verify the output formats
    assertTrue(outputFormats.get(operatorID2).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams1 = loadedFormatContainer.getParameters(operatorID2);
    assertEquals(1, outputFormatParams1.keySet().size());
    assertEquals("bcd234", outputFormatParams1.getString("parameter1", null));
    assertTrue(outputFormats.get(operatorID3).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams2 = loadedFormatContainer.getParameters(operatorID3);
    assertEquals(1, outputFormatParams2.keySet().size());
    assertEquals("cde345", outputFormatParams2.getString("parameter1", null));
}

Example 47 with DiscardingOutputFormat

use of org.apache.flink.api.java.io.DiscardingOutputFormat in project flink by apache.

the class ErrorHandlingSubmissionJob method main.

public static void main(String[] args) throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.fromCollection(Arrays.asList(1, 2, 3)).map(element -> element + 1).output(new DiscardingOutputFormat<>());
    try {
        env.execute();
    } catch (Exception e) {
        SUBMISSION_EXCEPTION.set(e);
        throw e;
    }
}

Example 48 with DiscardingOutputFormat

use of org.apache.flink.api.java.io.DiscardingOutputFormat in project flink by apache.

the class LocalExecutorITCase method getRuntimeExceptionPlan.

private Plan getRuntimeExceptionPlan() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.fromElements(1).map(element -> {
        if (element == 1) {
            throw new RuntimeException("oups");
        }
        return element;
    }).output(new DiscardingOutputFormat<>());
    return env.createProgramPlan();
}

Example 49 with DiscardingOutputFormat

use of org.apache.flink.api.java.io.DiscardingOutputFormat in project flink by apache.

the class WordCountCompilerTest method checkWordCount.

private void checkWordCount(boolean estimates) {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // get input data
    DataSet<String> lines = env.readTextFile(IN_FILE).name("Input Lines");
    lines.map(new MapFunction<String, Tuple2<String, Integer>>() {

        private static final long serialVersionUID = -3952739820618875030L;

        @Override
        public Tuple2<String, Integer> map(String v) throws Exception {
            return new Tuple2<>(v, 1);
        }
    }).name("Tokenize Lines").groupBy(0).sum(1).name("Count Words").output(new DiscardingOutputFormat<Tuple2<String, Integer>>()).name("Word Counts");
    // get the plan and compile it
    Plan p = env.createProgramPlan();
    p.setExecutionConfig(new ExecutionConfig());
    OptimizedPlan plan;
    if (estimates) {
        GenericDataSourceBase<?, ?> source = getContractResolver(p).getNode("Input Lines");
        setSourceStatistics(source, 1024 * 1024 * 1024 * 1024L, 24f);
        plan = compileWithStats(p);
    } else {
        plan = compileNoStats(p);
    }
    // get the optimizer plan nodes
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(plan);
    SinkPlanNode sink = resolver.getNode("Word Counts");
    SingleInputPlanNode reducer = resolver.getNode("Count Words");
    SingleInputPlanNode mapper = resolver.getNode("Tokenize Lines");
    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Channel c = reducer.getInput();
    Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
    FieldList l = new FieldList(0);
    Assert.assertEquals(l, c.getShipStrategyKeys());
    Assert.assertEquals(l, c.getLocalStrategyKeys());
    Assert.assertTrue(Arrays.equals(c.getLocalStrategySortOrder(), reducer.getSortOrders(0)));
    // check the combiner
    SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
    Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
    Assert.assertEquals(l, combiner.getKeys(0));
    Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
}