Examples with FoldFunction org.apache.flink.api.common.functions.FoldFunction used on opensource projects

Example 1 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class WindowFoldITCase method testFoldProcessWindow.

@Test
public void testFoldProcessWindow() throws Exception {
    testResults = new ArrayList<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(1);
    DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            ctx.collect(Tuple2.of("a", 0));
            ctx.collect(Tuple2.of("a", 1));
            ctx.collect(Tuple2.of("a", 2));
            ctx.collect(Tuple2.of("b", 3));
            ctx.collect(Tuple2.of("b", 4));
            ctx.collect(Tuple2.of("b", 5));
            ctx.collect(Tuple2.of("a", 6));
            ctx.collect(Tuple2.of("a", 7));
            ctx.collect(Tuple2.of("a", 8));
        // source is finite, so it will have an implicit MAX watermark when it finishes
        }

        @Override
        public void cancel() {
        }
    }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
    source1.keyBy(0).window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))).fold(Tuple2.of(0, "R:"), new FoldFunction<Tuple2<String, Integer>, Tuple2<Integer, String>>() {

        @Override
        public Tuple2<Integer, String> fold(Tuple2<Integer, String> accumulator, Tuple2<String, Integer> value) throws Exception {
            accumulator.f1 += value.f0;
            accumulator.f0 += value.f1;
            return accumulator;
        }
    }, new ProcessWindowFunction<Tuple2<Integer, String>, Tuple3<String, Integer, Integer>, Tuple, TimeWindow>() {

        @Override
        public void process(Tuple tuple, Context context, Iterable<Tuple2<Integer, String>> elements, Collector<Tuple3<String, Integer, Integer>> out) throws Exception {
            int i = 0;
            for (Tuple2<Integer, String> in : elements) {
                out.collect(new Tuple3<>(in.f1, in.f0, i++));
            }
        }
    }).addSink(new SinkFunction<Tuple3<String, Integer, Integer>>() {

        @Override
        public void invoke(Tuple3<String, Integer, Integer> value) throws Exception {
            testResults.add(value.toString());
        }
    });
    env.execute("Fold Process Window Test");
    List<String> expectedResult = Arrays.asList("(R:aaa,3,0)", "(R:aaa,21,0)", "(R:bbb,12,0)");
    Collections.sort(expectedResult);
    Collections.sort(testResults);
    Assert.assertEquals(expectedResult, testResults);
}

Example 2 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class WindowFoldITCase method testFoldProcessAllWindow.

@Test
public void testFoldProcessAllWindow() throws Exception {
    testResults = new ArrayList<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(1);
    DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            ctx.collect(Tuple2.of("a", 0));
            ctx.collect(Tuple2.of("a", 1));
            ctx.collect(Tuple2.of("a", 2));
            ctx.collect(Tuple2.of("b", 3));
            ctx.collect(Tuple2.of("b", 4));
            ctx.collect(Tuple2.of("b", 5));
            ctx.collect(Tuple2.of("a", 6));
            ctx.collect(Tuple2.of("a", 7));
            ctx.collect(Tuple2.of("a", 8));
        // source is finite, so it will have an implicit MAX watermark when it finishes
        }

        @Override
        public void cancel() {
        }
    }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
    source1.windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))).fold(Tuple2.of(0, "R:"), new FoldFunction<Tuple2<String, Integer>, Tuple2<Integer, String>>() {

        @Override
        public Tuple2<Integer, String> fold(Tuple2<Integer, String> accumulator, Tuple2<String, Integer> value) throws Exception {
            accumulator.f1 += value.f0;
            accumulator.f0 += value.f1;
            return accumulator;
        }
    }, new ProcessAllWindowFunction<Tuple2<Integer, String>, Tuple3<String, Integer, Integer>, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Tuple2<Integer, String>> elements, Collector<Tuple3<String, Integer, Integer>> out) throws Exception {
            int i = 0;
            for (Tuple2<Integer, String> in : elements) {
                out.collect(new Tuple3<>(in.f1, in.f0, i++));
            }
        }
    }).addSink(new SinkFunction<Tuple3<String, Integer, Integer>>() {

        @Override
        public void invoke(Tuple3<String, Integer, Integer> value) throws Exception {
            testResults.add(value.toString());
        }
    });
    env.execute("Fold Process Window Test");
    List<String> expectedResult = Arrays.asList("(R:aaa,3,0)", "(R:aaa,21,0)", "(R:bbb,12,0)");
    Collections.sort(expectedResult);
    Collections.sort(testResults);
    Assert.assertEquals(expectedResult, testResults);
}

Example 3 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class RichAsyncFunctionTest method testRuntimeContext.

/**
	 * Test the set of runtime context methods in the context of a {@link RichAsyncFunction}.
	 */
@Test
public void testRuntimeContext() throws Exception {
    RichAsyncFunction<Integer, Integer> function = new RichAsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1707630162838967972L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
        // no op
        }
    };
    final String taskName = "foobarTask";
    final MetricGroup metricGroup = mock(MetricGroup.class);
    final int numberOfParallelSubtasks = 42;
    final int indexOfSubtask = 43;
    final int attemptNumber = 1337;
    final String taskNameWithSubtask = "barfoo";
    final ExecutionConfig executionConfig = mock(ExecutionConfig.class);
    final ClassLoader userCodeClassLoader = mock(ClassLoader.class);
    RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
    when(mockedRuntimeContext.getTaskName()).thenReturn(taskName);
    when(mockedRuntimeContext.getMetricGroup()).thenReturn(metricGroup);
    when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(numberOfParallelSubtasks);
    when(mockedRuntimeContext.getIndexOfThisSubtask()).thenReturn(indexOfSubtask);
    when(mockedRuntimeContext.getAttemptNumber()).thenReturn(attemptNumber);
    when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenReturn(taskNameWithSubtask);
    when(mockedRuntimeContext.getExecutionConfig()).thenReturn(executionConfig);
    when(mockedRuntimeContext.getUserCodeClassLoader()).thenReturn(userCodeClassLoader);
    function.setRuntimeContext(mockedRuntimeContext);
    RuntimeContext runtimeContext = function.getRuntimeContext();
    assertEquals(taskName, runtimeContext.getTaskName());
    assertEquals(metricGroup, runtimeContext.getMetricGroup());
    assertEquals(numberOfParallelSubtasks, runtimeContext.getNumberOfParallelSubtasks());
    assertEquals(indexOfSubtask, runtimeContext.getIndexOfThisSubtask());
    assertEquals(attemptNumber, runtimeContext.getAttemptNumber());
    assertEquals(taskNameWithSubtask, runtimeContext.getTaskNameWithSubtasks());
    assertEquals(executionConfig, runtimeContext.getExecutionConfig());
    assertEquals(userCodeClassLoader, runtimeContext.getUserCodeClassLoader());
    try {
        runtimeContext.getDistributedCache();
        fail("Expected getDistributedCached to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getState(new ValueStateDescriptor<>("foobar", Integer.class, 42));
        fail("Expected getState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getListState(new ListStateDescriptor<>("foobar", Integer.class));
        fail("Expected getListState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getReducingState(new ReducingStateDescriptor<>("foobar", new ReduceFunction<Integer>() {

            private static final long serialVersionUID = 2136425961884441050L;

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1;
            }
        }, Integer.class));
        fail("Expected getReducingState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getFoldingState(new FoldingStateDescriptor<>("foobar", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator;
            }
        }, Integer.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getMapState(new MapStateDescriptor<>("foobar", Integer.class, String.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.addAccumulator("foobar", new Accumulator<Integer, Integer>() {

            private static final long serialVersionUID = -4673320336846482358L;

            @Override
            public void add(Integer value) {
            // no op
            }

            @Override
            public Integer getLocalValue() {
                return null;
            }

            @Override
            public void resetLocal() {
            }

            @Override
            public void merge(Accumulator<Integer, Integer> other) {
            }

            @Override
            public Accumulator<Integer, Integer> clone() {
                return null;
            }
        });
        fail("Expected addAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAccumulator("foobar");
        fail("Expected getAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAllAccumulators();
        fail("Expected getAllAccumulators to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getIntCounter("foobar");
        fail("Expected getIntCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getLongCounter("foobar");
        fail("Expected getLongCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getDoubleCounter("foobar");
        fail("Expected getDoubleCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getHistogram("foobar");
        fail("Expected getHistogram to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.hasBroadcastVariable("foobar");
        fail("Expected hasBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariable("foobar");
        fail("Expected getBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariableWithInitializer("foobar", new BroadcastVariableInitializer<Object, Object>() {

            @Override
            public Object initializeBroadcastVariable(Iterable<Object> data) {
                return null;
            }
        });
        fail("Expected getBroadcastVariableWithInitializer to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}

Example 4 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class StreamingRuntimeContextTest method testFoldingStateInstantiation.

@Test
public void testFoldingStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = new StreamingRuntimeContext(createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap());
    @SuppressWarnings("unchecked") FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class);
    FoldingStateDescriptor<String, TaskInfo> descr = new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class);
    context.getFoldingState(descr);
    FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}

Example 5 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class WindowFoldITCase method testFoldWindow.

@Test
public void testFoldWindow() throws Exception {
    testResults = new ArrayList<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(1);
    DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            ctx.collect(Tuple2.of("a", 0));
            ctx.collect(Tuple2.of("a", 1));
            ctx.collect(Tuple2.of("a", 2));
            ctx.collect(Tuple2.of("b", 3));
            ctx.collect(Tuple2.of("b", 4));
            ctx.collect(Tuple2.of("b", 5));
            ctx.collect(Tuple2.of("a", 6));
            ctx.collect(Tuple2.of("a", 7));
            ctx.collect(Tuple2.of("a", 8));
        // source is finite, so it will have an implicit MAX watermark when it finishes
        }

        @Override
        public void cancel() {
        }
    }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
    source1.keyBy(0).window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))).fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {

        @Override
        public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
            accumulator.f0 += value.f0;
            accumulator.f1 += value.f1;
            return accumulator;
        }
    }).addSink(new SinkFunction<Tuple2<String, Integer>>() {

        @Override
        public void invoke(Tuple2<String, Integer> value) throws Exception {
            testResults.add(value.toString());
        }
    });
    env.execute("Fold Window Test");
    List<String> expectedResult = Arrays.asList("(R:aaa,3)", "(R:aaa,21)", "(R:bbb,12)");
    Collections.sort(expectedResult);
    Collections.sort(testResults);
    Assert.assertEquals(expectedResult, testResults);
}