Examples with ReducingStateDescriptor org.apache.flink.api.common.state.ReducingStateDescriptor used on opensource projects

Example 31 with ReducingStateDescriptor

use of org.apache.flink.api.common.state.ReducingStateDescriptor in project flink by apache.

the class RocksDBReducingStateTest method testAddAndGet.

// ------------------------------------------------------------------------
@Test
public void testAddAndGet() throws Exception {
    final ReducingStateDescriptor<Long> stateDescr = new ReducingStateDescriptor<>("my-state", new AddingFunction(), Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final RocksDBStateBackend backend = new RocksDBStateBackend(tmp.newFolder().toURI());
    backend.setDbStoragePath(tmp.newFolder().getAbsolutePath());
    final RocksDBKeyedStateBackend<String> keyedBackend = createKeyedBackend(backend);
    try {
        InternalReducingState<VoidNamespace, Long> state = keyedBackend.createReducingState(VoidNamespaceSerializer.INSTANCE, stateDescr);
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        state.add(17L);
        state.add(11L);
        assertEquals(28L, state.get().longValue());
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertNull(state.get());
        state.add(1L);
        state.add(2L);
        keyedBackend.setCurrentKey("def");
        assertEquals(28L, state.get().longValue());
        state.clear();
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        state.add(3L);
        state.add(2L);
        state.add(1L);
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertEquals(9L, state.get().longValue());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 32 with ReducingStateDescriptor

use of org.apache.flink.api.common.state.ReducingStateDescriptor in project flink by apache.

the class HeapReducingStateTest method testAddAndGet.

@Test
public void testAddAndGet() throws Exception {
    final ReducingStateDescriptor<Long> stateDescr = new ReducingStateDescriptor<>("my-state", new AddingFunction(), Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend();
    try {
        InternalReducingState<VoidNamespace, Long> state = keyedBackend.createReducingState(VoidNamespaceSerializer.INSTANCE, stateDescr);
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        state.add(17L);
        state.add(11L);
        assertEquals(28L, state.get().longValue());
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertNull(state.get());
        state.add(1L);
        state.add(2L);
        keyedBackend.setCurrentKey("def");
        assertEquals(28L, state.get().longValue());
        state.clear();
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        state.add(3L);
        state.add(2L);
        state.add(1L);
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertEquals(9L, state.get().longValue());
        state.clear();
        // make sure all lists / maps are cleared
        StateTable<String, VoidNamespace, Long> stateTable = ((HeapReducingState<String, VoidNamespace, Long>) state).stateTable;
        assertTrue(stateTable.isEmpty());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 33 with ReducingStateDescriptor

use of org.apache.flink.api.common.state.ReducingStateDescriptor in project flink by apache.

the class WindowedStream method apply.

/**
	 * Applies the given window function to each window. The window function is called for each
	 * evaluation of the window for each key individually. The output of the window function is
	 * interpreted as a regular non-windowed stream.
	 *
	 * <p>
	 * Arriving data is incrementally aggregated using the given reducer.
	 *
	 * @param reduceFunction The reduce function that is used for incremental aggregation.
	 * @param function The window function.
	 * @param resultType Type information for the result type of the window function
	 * @return The data stream that is the result of applying the window function to the window.
	 *
	 * @deprecated Use {@link #reduce(ReduceFunction, WindowFunction, TypeInformation)} instead.
	 */
@Deprecated
public <R> SingleOutputStreamOperator<R> apply(ReduceFunction<T> reduceFunction, WindowFunction<T, R, K, W> function, TypeInformation<R> resultType) {
    if (reduceFunction instanceof RichFunction) {
        throw new UnsupportedOperationException("ReduceFunction of apply can not be a RichFunction.");
    }
    //clean the closures
    function = input.getExecutionEnvironment().clean(function);
    reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);
    String callLocation = Utils.getCallLocationName();
    String udfName = "WindowedStream." + callLocation;
    String opName;
    KeySelector<T, K> keySel = input.getKeySelector();
    OneInputStreamOperator<T, R> operator;
    if (evictor != null) {
        @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<T>> streamRecordSerializer = (TypeSerializer<StreamRecord<T>>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        ListStateDescriptor<StreamRecord<T>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer);
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
        operator = new EvictingWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalIterableWindowFunction<>(new ReduceApplyWindowFunction<>(reduceFunction, function)), trigger, evictor, allowedLateness, lateDataOutputTag);
    } else {
        ReducingStateDescriptor<T> stateDesc = new ReducingStateDescriptor<>("window-contents", reduceFunction, input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
        operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalSingleValueWindowFunction<>(function), trigger, allowedLateness, lateDataOutputTag);
    }
    return input.transform(opName, resultType, operator);
}

Example 34 with ReducingStateDescriptor

use of org.apache.flink.api.common.state.ReducingStateDescriptor in project flink by apache.

the class WindowedStream method reduce.

/**
	 * Applies the given window function to each window. The window function is called for each
	 * evaluation of the window for each key individually. The output of the window function is
	 * interpreted as a regular non-windowed stream.
	 *
	 * <p>Arriving data is incrementally aggregated using the given reducer.
	 *
	 * @param reduceFunction The reduce function that is used for incremental aggregation.
	 * @param function The window function.
	 * @param resultType Type information for the result type of the window function
	 * @return The data stream that is the result of applying the window function to the window.
	 */
@Internal
public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction, ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) {
    if (reduceFunction instanceof RichFunction) {
        throw new UnsupportedOperationException("ReduceFunction of apply can not be a RichFunction.");
    }
    //clean the closures
    function = input.getExecutionEnvironment().clean(function);
    reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);
    String callLocation = Utils.getCallLocationName();
    String udfName = "WindowedStream." + callLocation;
    String opName;
    KeySelector<T, K> keySel = input.getKeySelector();
    OneInputStreamOperator<T, R> operator;
    if (evictor != null) {
        @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<T>> streamRecordSerializer = (TypeSerializer<StreamRecord<T>>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        ListStateDescriptor<StreamRecord<T>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer);
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
        operator = new EvictingWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalIterableProcessWindowFunction<>(new ReduceApplyProcessWindowFunction<>(reduceFunction, function)), trigger, evictor, allowedLateness, lateDataOutputTag);
    } else {
        ReducingStateDescriptor<T> stateDesc = new ReducingStateDescriptor<>("window-contents", reduceFunction, input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
        operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalSingleValueProcessWindowFunction<>(function), trigger, allowedLateness, lateDataOutputTag);
    }
    return input.transform(opName, resultType, operator);
}

Example 35 with ReducingStateDescriptor

use of org.apache.flink.api.common.state.ReducingStateDescriptor in project flink by apache.

the class StateBackendTestBase method testReducingStateDefaultValue.

/**
	 * Verify that an empty {@code ReduceState} yields {@code null}.
	 */
@Test
public void testReducingStateDefaultValue() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(1);
    assertNull(state.get());
    state.add("Ciao");
    assertEquals("Ciao", state.get());
    state.clear();
    assertNull(state.get());
    backend.dispose();
}