Examples with ListState org.apache.flink.api.common.state.ListState used on opensource projects

Example 6 with ListState

use of org.apache.flink.api.common.state.ListState in project beam by apache.

the class DoFnOperator method initializeState.

@Override
public void initializeState(StateInitializationContext context) throws Exception {
    super.initializeState(context);
    ListStateDescriptor<WindowedValue<InputT>> pushedBackStateDescriptor = new ListStateDescriptor<>("pushed-back-elements", new CoderTypeSerializer<>(windowedInputCoder, serializedOptions));
    if (keySelector != null) {
        pushedBackElementsHandler = KeyedPushedBackElementsHandler.create(keySelector, getKeyedStateBackend(), pushedBackStateDescriptor);
    } else {
        ListState<WindowedValue<InputT>> listState = getOperatorStateBackend().getListState(pushedBackStateDescriptor);
        pushedBackElementsHandler = NonKeyedPushedBackElementsHandler.create(listState);
    }
    currentInputWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis();
    currentSideInputWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis();
    currentOutputWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis();
    sideInputReader = NullSideInputReader.of(sideInputs);
    if (!sideInputs.isEmpty()) {
        FlinkBroadcastStateInternals sideInputStateInternals = new FlinkBroadcastStateInternals<>(getContainingTask().getIndexInSubtaskGroup(), getOperatorStateBackend(), serializedOptions);
        sideInputHandler = new SideInputHandler(sideInputs, sideInputStateInternals);
        sideInputReader = sideInputHandler;
        Stream<WindowedValue<InputT>> pushedBack = pushedBackElementsHandler.getElements();
        long min = pushedBack.map(v -> v.getTimestamp().getMillis()).reduce(Long.MAX_VALUE, Math::min);
        pushedBackWatermark = min;
    } else {
        pushedBackWatermark = Long.MAX_VALUE;
    }
    // StatefulPardo or WindowDoFn
    if (keyCoder != null) {
        keyedStateInternals = new FlinkStateInternals<>((KeyedStateBackend) getKeyedStateBackend(), keyCoder, serializedOptions);
        if (timerService == null) {
            timerService = getInternalTimerService("beam-timer", new CoderTypeSerializer<>(timerCoder, serializedOptions), this);
        }
        timerInternals = new FlinkTimerInternals();
        timeServiceManagerCompat = getTimeServiceManagerCompat();
    }
    outputManager = outputManagerFactory.create(output, getLockToAcquireForStateAccessDuringBundles(), getOperatorStateBackend());
}

Example 7 with ListState

use of org.apache.flink.api.common.state.ListState in project flink by apache.

the class AbstractKeyedStateBackend method getOrCreateKeyedState.

/**
	 * @see KeyedStateBackend
	 */
@Override
public <N, S extends State, V> S getOrCreateKeyedState(final TypeSerializer<N> namespaceSerializer, StateDescriptor<S, V> stateDescriptor) throws Exception {
    checkNotNull(namespaceSerializer, "Namespace serializer");
    if (keySerializer == null) {
        throw new UnsupportedOperationException("State key serializer has not been configured in the config. " + "This operation cannot use partitioned state.");
    }
    if (!stateDescriptor.isSerializerInitialized()) {
        throw new IllegalStateException("The serializer of the descriptor has not been initialized!");
    }
    InternalKvState<?> existing = keyValueStatesByName.get(stateDescriptor.getName());
    if (existing != null) {
        @SuppressWarnings("unchecked") S typedState = (S) existing;
        return typedState;
    }
    // create a new blank key/value state
    S state = stateDescriptor.bind(new StateBinder() {

        @Override
        public <T> ValueState<T> createValueState(ValueStateDescriptor<T> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createValueState(namespaceSerializer, stateDesc);
        }

        @Override
        public <T> ListState<T> createListState(ListStateDescriptor<T> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createListState(namespaceSerializer, stateDesc);
        }

        @Override
        public <T> ReducingState<T> createReducingState(ReducingStateDescriptor<T> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createReducingState(namespaceSerializer, stateDesc);
        }

        @Override
        public <T, ACC, R> AggregatingState<T, R> createAggregatingState(AggregatingStateDescriptor<T, ACC, R> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createAggregatingState(namespaceSerializer, stateDesc);
        }

        @Override
        public <T, ACC> FoldingState<T, ACC> createFoldingState(FoldingStateDescriptor<T, ACC> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createFoldingState(namespaceSerializer, stateDesc);
        }

        @Override
        public <UK, UV> MapState<UK, UV> createMapState(MapStateDescriptor<UK, UV> stateDesc) throws Exception {
            return AbstractKeyedStateBackend.this.createMapState(namespaceSerializer, stateDesc);
        }
    });
    @SuppressWarnings("unchecked") InternalKvState<N> kvState = (InternalKvState<N>) state;
    keyValueStatesByName.put(stateDescriptor.getName(), kvState);
    // Publish queryable state
    if (stateDescriptor.isQueryable()) {
        if (kvStateRegistry == null) {
            throw new IllegalStateException("State backend has not been initialized for job.");
        }
        String name = stateDescriptor.getQueryableStateName();
        kvStateRegistry.registerKvState(keyGroupRange, name, kvState);
    }
    return state;
}

Example 8 with ListState

use of org.apache.flink.api.common.state.ListState in project flink by apache.

the class StreamArrowPythonGroupWindowAggregateFunctionOperator method open.

@Override
public void open() throws Exception {
    super.open();
    windowSerializer = windowAssigner.getWindowSerializer(new ExecutionConfig());
    internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    triggerContext = new TriggerContext();
    triggerContext.open();
    StateDescriptor<ListState<RowData>, List<RowData>> windowStateDescriptor = new ListStateDescriptor<>("window-input", new RowDataSerializer(inputType));
    StateDescriptor<ListState<RowData>, List<RowData>> dataRetractStateDescriptor = new ListStateDescriptor<>("data-retract", new RowDataSerializer(inputType));
    this.windowAccumulateData = (InternalListState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
    this.windowRetractData = (InternalListState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, dataRetractStateDescriptor);
    inputKeyAndWindow = new LinkedList<>();
    windowProperty = new GenericRowData(namedProperties.length);
    windowAggResult = new JoinedRowData();
    WindowContext windowContext = new WindowContext();
    windowAssigner.open(windowContext);
}

Example 9 with ListState

use of org.apache.flink.api.common.state.ListState in project flink by apache.

the class WindowReaderOperator method evictingWindow.

public static <KEY, T, W extends Window, OUT> WindowReaderOperator<?, KEY, StreamRecord<T>, W, OUT> evictingWindow(WindowReaderFunction<StreamRecord<T>, OUT, KEY, W> readerFunction, TypeInformation<KEY> keyType, TypeSerializer<W> windowSerializer, TypeInformation<T> stateType, ExecutionConfig config) {
    @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<T>> streamRecordSerializer = (TypeSerializer<StreamRecord<T>>) new StreamElementSerializer(stateType.createSerializer(config));
    StateDescriptor<ListState<StreamRecord<T>>, List<StreamRecord<T>>> descriptor = new ListStateDescriptor<>(WINDOW_STATE_NAME, streamRecordSerializer);
    return new WindowReaderOperator<>(readerFunction, keyType, windowSerializer, WindowContents.listState(), descriptor);
}

Example 10 with ListState

use of org.apache.flink.api.common.state.ListState in project beam by apache.

the class ImpulseSourceFunctionTest method getMockListState.

private static <T> ListState<T> getMockListState(List<T> initialState) throws Exception {
    ListState mock = Mockito.mock(ListState.class);
    when(mock.get()).thenReturn(initialState);
    return mock;
}