Search in sources :

Example 6 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class WindowFoldITCase method testFoldAllWindow.

@Test
public void testFoldAllWindow() throws Exception {
    testResults = new ArrayList<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(1);
    DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            ctx.collect(Tuple2.of("a", 0));
            ctx.collect(Tuple2.of("a", 1));
            ctx.collect(Tuple2.of("a", 2));
            ctx.collect(Tuple2.of("b", 3));
            ctx.collect(Tuple2.of("a", 3));
            ctx.collect(Tuple2.of("b", 4));
            ctx.collect(Tuple2.of("a", 4));
            ctx.collect(Tuple2.of("b", 5));
            ctx.collect(Tuple2.of("a", 5));
        // source is finite, so it will have an implicit MAX watermark when it finishes
        }

        @Override
        public void cancel() {
        }
    }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
    source1.windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))).fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {

        @Override
        public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
            accumulator.f0 += value.f0;
            accumulator.f1 += value.f1;
            return accumulator;
        }
    }).addSink(new SinkFunction<Tuple2<String, Integer>>() {

        @Override
        public void invoke(Tuple2<String, Integer> value) throws Exception {
            testResults.add(value.toString());
        }
    });
    env.execute("Fold All-Window Test");
    List<String> expectedResult = Arrays.asList("(R:aaa,3)", "(R:bababa,24)");
    Collections.sort(expectedResult);
    Collections.sort(testResults);
    Assert.assertEquals(expectedResult, testResults);
}
Also used : SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) FoldFunction(org.apache.flink.api.common.functions.FoldFunction) Tuple2(org.apache.flink.api.java.tuple.Tuple2) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) Test(org.junit.Test)

Example 7 with FoldFunction

use of org.apache.flink.api.common.functions.FoldFunction in project flink by apache.

the class StateBackendTestBase method testConcurrentMapIfQueryable.

/**
	 * Tests that {@link AbstractHeapState} instances respect the queryable
	 * flag and create concurrent variants for internal state structures.
	 */
@SuppressWarnings("unchecked")
protected void testConcurrentMapIfQueryable() throws Exception {
    final int numberOfKeyGroups = 1;
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, numberOfKeyGroups, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
    {
        // ValueState
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("value-state", Integer.class, -1);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.update(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ListState
        ListStateDescriptor<Integer> desc = new ListStateDescriptor<>("list-state", Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ListState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ReducingState
        ReducingStateDescriptor<Integer> desc = new ReducingStateDescriptor<>("reducing-state", new ReduceFunction<Integer>() {

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1 + value2;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ReducingState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // FoldingState
        FoldingStateDescriptor<Integer, Integer> desc = new FoldingStateDescriptor<>("folding-state", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator + value;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        FoldingState<Integer, Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // MapState
        MapStateDescriptor<Integer, String> desc = new MapStateDescriptor<>("map-state", Integer.class, String.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.put(121818273, "121818273");
        int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(1, numberOfKeyGroups);
        StateTable stateTable = ((AbstractHeapState) kvState).getStateTable();
        assertNotNull("State not set", stateTable.get(keyGroupIndex));
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    backend.dispose();
}
Also used : NestedMapsStateTable(org.apache.flink.runtime.state.heap.NestedMapsStateTable) StateTable(org.apache.flink.runtime.state.heap.StateTable) AbstractHeapState(org.apache.flink.runtime.state.heap.AbstractHeapState) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) FoldFunction(org.apache.flink.api.common.functions.FoldFunction) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) ReduceFunction(org.apache.flink.api.common.functions.ReduceFunction) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) FoldingStateDescriptor(org.apache.flink.api.common.state.FoldingStateDescriptor) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) ListState(org.apache.flink.api.common.state.ListState) ReducingStateDescriptor(org.apache.flink.api.common.state.ReducingStateDescriptor) MapState(org.apache.flink.api.common.state.MapState) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) ReducingState(org.apache.flink.api.common.state.ReducingState) ValueState(org.apache.flink.api.common.state.ValueState) InternalValueState(org.apache.flink.runtime.state.internal.InternalValueState) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) FoldingState(org.apache.flink.api.common.state.FoldingState)

Aggregations

FoldFunction (org.apache.flink.api.common.functions.FoldFunction)7 Test (org.junit.Test)6 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)4 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)4 SourceFunction (org.apache.flink.streaming.api.functions.source.SourceFunction)4 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)3 ReduceFunction (org.apache.flink.api.common.functions.ReduceFunction)2 FoldingStateDescriptor (org.apache.flink.api.common.state.FoldingStateDescriptor)2 Tuple3 (org.apache.flink.api.java.tuple.Tuple3)2 Collector (org.apache.flink.util.Collector)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 TaskInfo (org.apache.flink.api.common.TaskInfo)1 Accumulator (org.apache.flink.api.common.accumulators.Accumulator)1 IterationRuntimeContext (org.apache.flink.api.common.functions.IterationRuntimeContext)1 RuntimeContext (org.apache.flink.api.common.functions.RuntimeContext)1 FoldingState (org.apache.flink.api.common.state.FoldingState)1 ListState (org.apache.flink.api.common.state.ListState)1 ListStateDescriptor (org.apache.flink.api.common.state.ListStateDescriptor)1 MapState (org.apache.flink.api.common.state.MapState)1 MapStateDescriptor (org.apache.flink.api.common.state.MapStateDescriptor)1