Examples with MapStateDescriptor org.apache.flink.api.common.state.MapStateDescriptor used on opensource projects

Example 41 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}

Example 42 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class LatencyMarkerITCase method testBroadcast.

/**
 * FLINK-17780: Tests that streams are not corrupted/records lost when using latency markers
 * with broadcast.
 */
@Test
public void testBroadcast() throws Exception {
    int inputCount = 100000;
    int parallelism = 4;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parallelism);
    env.getConfig().setLatencyTrackingInterval(2000);
    env.setRestartStrategy(RestartStrategies.noRestart());
    List<Integer> broadcastData = IntStream.range(0, inputCount).boxed().collect(Collectors.toList());
    DataStream<Integer> broadcastDataStream = env.fromCollection(broadcastData).setParallelism(1);
    // broadcast the configurations and create the broadcast state
    DataStream<String> streamWithoutData = env.fromCollection(Collections.emptyList(), TypeInformation.of(String.class));
    MapStateDescriptor<String, Integer> stateDescriptor = new MapStateDescriptor<>("BroadcastState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    SingleOutputStreamOperator<Integer> processor = streamWithoutData.connect(broadcastDataStream.broadcast(stateDescriptor)).process(new BroadcastProcessFunction<String, Integer, Integer>() {

        int expected = 0;

        public void processElement(String value, ReadOnlyContext ctx, Collector<Integer> out) {
        }

        public void processBroadcastElement(Integer value, Context ctx, Collector<Integer> out) {
            if (value != expected++) {
                throw new AssertionError(String.format("Value was supposed to be: '%s', but was: '%s'", expected - 1, value));
            }
            out.collect(value);
        }
    });
    processor.addSink(new AccumulatorCountingSink<>()).setParallelism(1);
    JobExecutionResult executionResult = env.execute();
    Integer count = executionResult.getAccumulatorResult(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR);
    Assert.assertEquals(inputCount * parallelism, count.intValue());
}

Example 43 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class ProcTimeRowsBoundedPrecedingFunction method open.

@Override
public void open(Configuration parameters) throws Exception {
    function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
    function.open(new PerKeyStateDataViewStore(getRuntimeContext()));
    output = new JoinedRowData();
    // input element are all binary row as they are came from network
    InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
    // We keep the elements received in a Map state keyed
    // by the ingestion time in the operator.
    // we also keep counter of processed elements
    // and timestamp of oldest element
    ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
    MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<Long, List<RowData>>("inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
    inputState = getRuntimeContext().getMapState(mapStateDescriptor);
    InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
    ValueStateDescriptor<RowData> stateDescriptor = new ValueStateDescriptor<RowData>("accState", accTypeInfo);
    accState = getRuntimeContext().getState(stateDescriptor);
    ValueStateDescriptor<Long> processedCountDescriptor = new ValueStateDescriptor<Long>("processedCountState", Types.LONG);
    counterState = getRuntimeContext().getState(processedCountDescriptor);
    ValueStateDescriptor<Long> smallestTimestampDescriptor = new ValueStateDescriptor<Long>("smallestTSState", Types.LONG);
    smallestTsState = getRuntimeContext().getState(smallestTimestampDescriptor);
    initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime");
}

Example 44 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class RowTimeRangeBoundedPrecedingFunction method open.

@Override
public void open(Configuration parameters) throws Exception {
    function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
    function.open(new PerKeyStateDataViewStore(getRuntimeContext()));
    output = new JoinedRowData();
    ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>("lastTriggeringTsState", Types.LONG);
    lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);
    InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
    ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo);
    accState = getRuntimeContext().getState(accStateDesc);
    // input element are all binary row as they are came from network
    InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
    ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
    MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>("inputState", Types.LONG, rowListTypeInfo);
    inputState = getRuntimeContext().getMapState(inputStateDesc);
    ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>("cleanupTsState", Types.LONG);
    this.cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor);
    // metrics
    this.numLateRecordsDropped = getRuntimeContext().getMetricGroup().counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
}

Example 45 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class TemporalRowTimeJoinOperator method open.

@Override
public void open() throws Exception {
    super.open();
    joinCondition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
    joinCondition.setRuntimeContext(getRuntimeContext());
    joinCondition.open(new Configuration());
    nextLeftIndex = getRuntimeContext().getState(new ValueStateDescriptor<>(NEXT_LEFT_INDEX_STATE_NAME, Types.LONG));
    leftState = getRuntimeContext().getMapState(new MapStateDescriptor<>(LEFT_STATE_NAME, Types.LONG, leftType));
    rightState = getRuntimeContext().getMapState(new MapStateDescriptor<>(RIGHT_STATE_NAME, Types.LONG, rightType));
    registeredTimer = getRuntimeContext().getState(new ValueStateDescriptor<>(REGISTERED_TIMER_STATE_NAME, Types.LONG));
    timerService = getInternalTimerService(TIMERS_STATE_NAME, VoidNamespaceSerializer.INSTANCE, this);
    outRow = new JoinedRowData();
    rightNullRow = new GenericRowData(rightType.toRowType().getFieldCount());
    collector = new TimestampedCollector<>(output);
}