Examples with MapStateDescriptor org.apache.flink.api.common.state.MapStateDescriptor used on opensource projects

Example 11 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class StreamGraphGeneratorTest method testUnalignedCheckpointDisabledOnBroadcast.

@Test
public void testUnalignedCheckpointDisabledOnBroadcast() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(42);
    DataStream<Long> source1 = env.fromSequence(1L, 10L);
    DataStream<Long> map1 = source1.broadcast().map(l -> l);
    DataStream<Long> source2 = env.fromSequence(2L, 11L);
    DataStream<Long> keyed = source2.keyBy(r -> 0L);
    final MapStateDescriptor<Long, Long> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final BroadcastStream<Long> broadcast = map1.broadcast(descriptor);
    final SingleOutputStreamOperator<Long> joined = keyed.connect(broadcast).process(new KeyedBroadcastProcessFunction<Long, Long, Long, Long>() {

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<Long> out) {
        }

        @Override
        public void processBroadcastElement(Long value, Context ctx, Collector<Long> out) {
        }
    });
    StreamGraph streamGraph = env.getStreamGraph();
    assertEquals(4, streamGraph.getStreamNodes().size());
    // single broadcast
    assertThat(edge(streamGraph, source1, map1), supportsUnalignedCheckpoints(false));
    // keyed, connected with broadcast
    assertThat(edge(streamGraph, source2, joined), supportsUnalignedCheckpoints(false));
    // broadcast, connected with keyed
    assertThat(edge(streamGraph, map1, joined), supportsUnalignedCheckpoints(false));
}

Example 12 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class SavepointStateBackendSwitchTestBase method switchStateBackend.

@Test
public void switchStateBackend() throws Exception {
    final File pathToWrite = tempFolder.newFile();
    final MapStateDescriptor<Long, Long> mapStateDescriptor = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
    mapStateDescriptor.initializeSerializerUnlessSet(new ExecutionConfig());
    final ValueStateDescriptor<Long> valueStateDescriptor = new ValueStateDescriptor<>("my-value-state", Long.class);
    valueStateDescriptor.initializeSerializerUnlessSet(new ExecutionConfig());
    final ListStateDescriptor<Long> listStateDescriptor = new ListStateDescriptor<>("my-list-state", Long.class);
    listStateDescriptor.initializeSerializerUnlessSet(new ExecutionConfig());
    final Integer namespace1 = 1;
    final Integer namespace2 = 2;
    final Integer namespace3 = 3;
    final Integer namespace4 = 4;
    try (final CheckpointableKeyedStateBackend<String> keyedBackend = fromBackend.createBackend(KEY_GROUP_RANGE, NUM_KEY_GROUPS, Collections.emptyList())) {
        takeSavepoint(keyedBackend, pathToWrite, mapStateDescriptor, valueStateDescriptor, listStateDescriptor, namespace1, namespace2, namespace3, namespace4);
    }
    final SnapshotResult<KeyedStateHandle> stateHandles;
    try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(pathToWrite)))) {
        stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
    }
    final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
    try (final CheckpointableKeyedStateBackend<String> keyedBackend = toBackend.createBackend(KEY_GROUP_RANGE, NUM_KEY_GROUPS, StateObjectCollection.singleton(stateHandle))) {
        verifyRestoredState(mapStateDescriptor, valueStateDescriptor, listStateDescriptor, namespace1, namespace2, namespace3, namespace4, keyedBackend);
    }
}

Example 13 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class BroadcastStateITCase method testBroadcastTranslation.

@Test
public void testBroadcastTranslation() throws Exception {
    final MapStateDescriptor<Long, String> utterDescriptor = new MapStateDescriptor<>("broadcast-state", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final Map<Long, String> expected = new HashMap<>();
    expected.put(0L, "test:0");
    expected.put(1L, "test:1");
    expected.put(2L, "test:2");
    expected.put(3L, "test:3");
    expected.put(4L, "test:4");
    expected.put(5L, "test:5");
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        private static final long serialVersionUID = -8500904795760316195L;

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    });
    final DataStream<String> srcTwo = env.fromCollection(expected.values()).assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        private static final long serialVersionUID = -2148318224248467213L;

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    final BroadcastStream<String> broadcast = srcTwo.broadcast(utterDescriptor);
    // the timestamp should be high enough to trigger the timer after all the elements arrive.
    final DataStream<String> output = srcOne.connect(broadcast).process(new TestBroadcastProcessFunction());
    output.addSink(new TestSink(0)).setParallelism(1);
    env.execute();
}

Example 14 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class BroadcastStateITCase method testKeyedWithBroadcastTranslation.

@Test
public void testKeyedWithBroadcastTranslation() throws Exception {
    final MapStateDescriptor<Long, String> utterDescriptor = new MapStateDescriptor<>("broadcast-state", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final Map<Long, String> expected = new HashMap<>();
    expected.put(0L, "test:0");
    expected.put(1L, "test:1");
    expected.put(2L, "test:2");
    expected.put(3L, "test:3");
    expected.put(4L, "test:4");
    expected.put(5L, "test:5");
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        private static final long serialVersionUID = -8500904795760316195L;

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromCollection(expected.values()).assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        private static final long serialVersionUID = -2148318224248467213L;

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    final BroadcastStream<String> broadcast = srcTwo.broadcast(utterDescriptor);
    // the timestamp should be high enough to trigger the timer after all the elements arrive.
    final DataStream<String> output = srcOne.connect(broadcast).process(new TestKeyedBroadcastProcessFunction(100000L, expected));
    output.addSink(new TestSink(expected.size())).setParallelism(1);
    env.execute();
}

Example 15 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class SortingBoundedInputITCase method testBatchExecutionWithTimersOneInput.

@Test
public void testBatchExecutionWithTimersOneInput() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // set parallelism to 1 to have consistent order of results
    env.setParallelism(1);
    Configuration config = new Configuration();
    config.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH);
    env.configure(config, this.getClass().getClassLoader());
    WatermarkStrategy<Tuple2<Integer, Integer>> watermarkStrategy = WatermarkStrategy.forGenerator(ctx -> GENERATE_WATERMARK_AFTER_4_14_TIMESTAMP).withTimestampAssigner((r, previousTimestamp) -> r.f1);
    SingleOutputStreamOperator<Tuple2<Integer, Integer>> elements = env.fromElements(Tuple2.of(1, 3), Tuple2.of(1, 1), Tuple2.of(2, 1), Tuple2.of(1, 4), // late element
    Tuple2.of(2, 3), // late element
    Tuple2.of(1, 2), Tuple2.of(1, 13), Tuple2.of(1, 11), Tuple2.of(2, 14), // late element
    Tuple2.of(1, 11)).assignTimestampsAndWatermarks(watermarkStrategy);
    OutputTag<Integer> lateElements = new OutputTag<>("late_elements", BasicTypeInfo.INT_TYPE_INFO);
    SingleOutputStreamOperator<Tuple3<Long, Integer, Integer>> sums = elements.map(element -> element.f0).keyBy(element -> element).process(new KeyedProcessFunction<Integer, Integer, Tuple3<Long, Integer, Integer>>() {

        private MapState<Long, Integer> countState;

        private ValueState<Long> previousTimestampState;

        @Override
        public void open(Configuration parameters) {
            countState = getRuntimeContext().getMapState(new MapStateDescriptor<>("sum", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO));
            previousTimestampState = getRuntimeContext().getState(new ValueStateDescriptor<>("previousTimestamp", BasicTypeInfo.LONG_TYPE_INFO));
        }

        @Override
        public void processElement(Integer value, Context ctx, Collector<Tuple3<Long, Integer, Integer>> out) throws Exception {
            Long elementTimestamp = ctx.timestamp();
            long nextTen = ((elementTimestamp + 10) / 10) * 10;
            ctx.timerService().registerEventTimeTimer(nextTen);
            if (elementTimestamp < ctx.timerService().currentWatermark()) {
                ctx.output(lateElements, value);
            } else {
                Long previousTimestamp = Optional.ofNullable(previousTimestampState.value()).orElse(0L);
                assertThat(elementTimestamp, greaterThanOrEqualTo(previousTimestamp));
                previousTimestampState.update(elementTimestamp);
                Integer currentCount = Optional.ofNullable(countState.get(nextTen)).orElse(0);
                countState.put(nextTen, currentCount + 1);
            }
        }

        @Override
        public void onTimer(long timestamp, OnTimerContext ctx, Collector<Tuple3<Long, Integer, Integer>> out) throws Exception {
            out.collect(Tuple3.of(timestamp, ctx.getCurrentKey(), countState.get(timestamp)));
            countState.remove(timestamp);
            // this would go in infinite loop if we did not quiesce the
            // timer service.
            ctx.timerService().registerEventTimeTimer(timestamp + 1);
        }
    });
    DataStream<Integer> lateStream = sums.getSideOutput(lateElements);
    List<Integer> lateRecordsCollected = CollectionUtil.iteratorToList(DataStreamUtils.collect(lateStream));
    List<Tuple3<Long, Integer, Integer>> sumsCollected = CollectionUtil.iteratorToList(DataStreamUtils.collect(sums));
    assertTrue(lateRecordsCollected.isEmpty());
    assertThat(sumsCollected, equalTo(Arrays.asList(Tuple3.of(10L, 1, 4), Tuple3.of(20L, 1, 3), Tuple3.of(10L, 2, 2), Tuple3.of(20L, 2, 1))));
}