Examples with RichReduceFunction org.apache.flink.api.common.functions.RichReduceFunction used on opensource projects

Example 6 with RichReduceFunction

use of org.apache.flink.api.common.functions.RichReduceFunction in project flink by apache.

the class ReduceCompilationTest method testAllReduceNoCombiner.

@Test
public void testAllReduceNoCombiner() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
        data.reduce(new RichReduceFunction<Double>() {

            @Override
            public Double reduce(Double value1, Double value2) {
                return value1 + value2;
            }
        }).name("reducer").output(new DiscardingOutputFormat<Double>()).name("sink");
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
        // the all-reduce has no combiner, when the parallelism of the input is one
        SourcePlanNode sourceNode = resolver.getNode("source");
        SingleInputPlanNode reduceNode = resolver.getNode("reducer");
        SinkPlanNode sinkNode = resolver.getNode("sink");
        // check wiring
        assertEquals(sourceNode, reduceNode.getInput().getSource());
        assertEquals(reduceNode, sinkNode.getInput().getSource());
        // check parallelism
        assertEquals(1, sourceNode.getParallelism());
        assertEquals(1, reduceNode.getParallelism());
        assertEquals(1, sinkNode.getParallelism());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
    }
}

Example 7 with RichReduceFunction

use of org.apache.flink.api.common.functions.RichReduceFunction in project flink by apache.

the class ReduceTranslationTests method translateGroupedReduceNoMapper.

@Test
public void translateGroupedReduceNoMapper() {
    try {
        final int parallelism = 8;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);
        initialData.groupBy(2).reduce(new RichReduceFunction<Tuple3<Double, StringValue, LongValue>>() {

            public Tuple3<Double, StringValue, LongValue> reduce(Tuple3<Double, StringValue, LongValue> value1, Tuple3<Double, StringValue, LongValue> value2) {
                return value1;
            }
        }).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        ReduceOperatorBase<?, ?> reducer = (ReduceOperatorBase<?, ?>) sink.getInput();
        // check types
        assertEquals(initialData.getType(), reducer.getOperatorInfo().getInputType());
        assertEquals(initialData.getType(), reducer.getOperatorInfo().getOutputType());
        // parallelism was not configured on the operator
        assertTrue(reducer.getParallelism() == parallelism || reducer.getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT);
        // check keys
        assertArrayEquals(new int[] { 2 }, reducer.getKeyColumns(0));
        assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}