Examples with MapStateDescriptor org.apache.flink.api.common.state.MapStateDescriptor used on opensource projects

Example 26 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class AbstractStreamArrowPythonOverWindowAggregateFunctionOperator method open.

@Override
public void open() throws Exception {
    super.open();
    InternalTimerService<VoidNamespace> internalTimerService = getInternalTimerService("python-over-window-timers", VoidNamespaceSerializer.INSTANCE, this);
    timerService = new SimpleTimerService(internalTimerService);
    InternalTypeInfo<RowData> inputTypeInfo = InternalTypeInfo.of(inputType);
    ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<>(inputTypeInfo);
    MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<>("inputState", Types.LONG, rowListTypeInfo);
    ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState", Types.LONG);
    lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);
    ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>("cleanupTsState", Types.LONG);
    cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor);
    inputState = getRuntimeContext().getMapState(inputStateDesc);
}

Example 27 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class AbstractQueryableStateTestBase method testMapState.

/**
 * Tests simple map state queryable state instance. Each source emits (subtaskIndex,
 * 0)..(subtaskIndex, numElements) tuples, which are then queried. The map state instance sums
 * the values up. The test succeeds after each subtask index is queried with result n*(n+1)/2.
 */
@Test
public void testMapState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    final MapStateDescriptor<Integer, Tuple2<Integer, Long>> mapStateDescriptor = new MapStateDescriptor<>("timon", BasicTypeInfo.INT_TYPE_INFO, source.getType());
    mapStateDescriptor.setQueryable("timon-queryable");
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 8470749712274833552L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).process(new ProcessFunction<Tuple2<Integer, Long>, Object>() {

        private static final long serialVersionUID = -805125545438296619L;

        private transient MapState<Integer, Tuple2<Integer, Long>> mapState;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            mapState = getRuntimeContext().getMapState(mapStateDescriptor);
        }

        @Override
        public void processElement(Tuple2<Integer, Long> value, Context ctx, Collector<Object> out) throws Exception {
            Tuple2<Integer, Long> v = mapState.get(value.f0);
            if (v == null) {
                v = new Tuple2<>(value.f0, 0L);
            }
            mapState.put(value.f0, new Tuple2<>(v.f0, v.f1 + value.f1));
        }
    });
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        final long expected = numElements * (numElements + 1L) / 2L;
        for (int key = 0; key < maxParallelism; key++) {
            boolean success = false;
            while (deadline.hasTimeLeft() && !success) {
                CompletableFuture<MapState<Integer, Tuple2<Integer, Long>>> future = getKvState(deadline, client, jobId, "timon-queryable", key, BasicTypeInfo.INT_TYPE_INFO, mapStateDescriptor, false, executor);
                Tuple2<Integer, Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get(key);
                if (value != null && value.f0 != null && expected == value.f1) {
                    assertEquals("Key mismatch", key, value.f0.intValue());
                    success = true;
                } else {
                    // Retry
                    Thread.sleep(RETRY_TIMEOUT);
                }
            }
            assertTrue("Did not succeed query", success);
        }
    }
}

Example 28 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class CEPITCase method testRichPatternSelectFunction.

@Test
public void testRichPatternSelectFunction() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(envConfiguration);
    env.setParallelism(2);
    DataStream<Event> input = env.fromElements(new Event(1, "barfoo", 1.0), new Event(2, "start", 2.0), new Event(3, "start", 2.1), new Event(3, "foobar", 3.0), new SubEvent(4, "foo", 4.0, 1.0), new SubEvent(3, "middle", 3.2, 1.0), new Event(42, "start", 3.1), new SubEvent(42, "middle", 3.3, 1.2), new Event(5, "middle", 5.0), new SubEvent(2, "middle", 6.0, 2.0), new SubEvent(7, "bar", 3.0, 3.0), new Event(42, "42", 42.0), new Event(3, "end", 2.0), new Event(2, "end", 1.0), new Event(42, "end", 42.0)).keyBy(new KeySelector<Event, Integer>() {

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    });
    Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new RichIterativeCondition<Event>() {

        @Override
        public boolean filter(Event value, Context<Event> ctx) throws Exception {
            return value.getName().equals("start");
        }
    }).followedByAny("middle").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {

        @Override
        public boolean filter(SubEvent value) throws Exception {
            return value.getName().equals("middle");
        }
    }).followedByAny("end").where(new SimpleCondition<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("end");
        }
    });
    DataStream<String> result = CEP.pattern(input, pattern).inProcessingTime().select(new RichPatternSelectFunction<Event, String>() {

        @Override
        public void open(Configuration config) {
            try {
                getRuntimeContext().getMapState(new MapStateDescriptor<>("test", LongSerializer.INSTANCE, LongSerializer.INSTANCE));
                throw new RuntimeException("Expected getMapState to fail with unsupported operation exception.");
            } catch (UnsupportedOperationException e) {
            // ignore, expected
            }
            getRuntimeContext().getUserCodeClassLoader();
        }

        @Override
        public String select(Map<String, List<Event>> p) throws Exception {
            StringBuilder builder = new StringBuilder();
            builder.append(p.get("start").get(0).getId()).append(",").append(p.get("middle").get(0).getId()).append(",").append(p.get("end").get(0).getId());
            return builder.toString();
        }
    });
    List<String> resultList = new ArrayList<>();
    DataStreamUtils.collect(result).forEachRemaining(resultList::add);
    resultList.sort(String::compareTo);
    assertEquals(Arrays.asList("2,2,2", "3,3,3", "42,42,42"), resultList);
}

Example 29 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class CepOperator method initializeState.

@Override
public void initializeState(StateInitializationContext context) throws Exception {
    super.initializeState(context);
    // initializeState through the provided context
    computationStates = context.getKeyedStateStore().getState(new ValueStateDescriptor<>(NFA_STATE_NAME, new NFAStateSerializer()));
    partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer, SharedBufferCacheConfig.of(getOperatorConfig().getConfiguration()));
    elementQueueState = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(EVENT_QUEUE_STATE_NAME, LongSerializer.INSTANCE, new ListSerializer<>(inputSerializer)));
    if (context.isRestored()) {
        partialMatches.migrateOldState(getKeyedStateBackend(), computationStates);
    }
}

Example 30 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class ExistingSavepoint method readBroadcastState.

/**
 * Read operator {@code BroadcastState} from a {@code Savepoint} when a custom serializer was
 * used; e.g., a different serializer than the one returned by {@code
 * TypeInformation#createSerializer}.
 *
 * @param uid The uid of the operator.
 * @param name The (unique) name for the state.
 * @param keyTypeInfo The type information for the keys in the state.
 * @param valueTypeInfo The type information for the values in the state.
 * @param keySerializer The type serializer used to write keys into the state.
 * @param valueSerializer The type serializer used to write values into the state.
 * @param <K> The type of keys in state.
 * @param <V> The type of values in state.
 * @return A {@code DataSet} of key-value pairs from state.
 * @throws IOException If the savepoint path is invalid or the uid does not exist.
 */
public <K, V> DataSource<Tuple2<K, V>> readBroadcastState(String uid, String name, TypeInformation<K> keyTypeInfo, TypeInformation<V> valueTypeInfo, TypeSerializer<K> keySerializer, TypeSerializer<V> valueSerializer) throws IOException {
    OperatorState operatorState = metadata.getOperatorState(uid);
    MapStateDescriptor<K, V> descriptor = new MapStateDescriptor<>(name, keySerializer, valueSerializer);
    BroadcastStateInputFormat<K, V> inputFormat = new BroadcastStateInputFormat<>(operatorState, env.getConfiguration(), stateBackend, descriptor);
    return env.createInput(inputFormat, new TupleTypeInfo<>(keyTypeInfo, valueTypeInfo));
}