Examples with FoldingStateDescriptor org.apache.flink.api.common.state.FoldingStateDescriptor used on opensource projects

Example 21 with FoldingStateDescriptor

use of org.apache.flink.api.common.state.FoldingStateDescriptor in project flink by apache.

the class AllWindowTranslationTest method testFoldEventTime.

// ------------------------------------------------------------------------
//  fold() translation tests
// ------------------------------------------------------------------------
@Test
@SuppressWarnings("rawtypes")
public void testFoldEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple3<String, String, Integer>> window1 = source.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))).fold(new Tuple3<>("", "", 1), new DummyFolder());
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 22 with FoldingStateDescriptor

use of org.apache.flink.api.common.state.FoldingStateDescriptor in project flink by apache.

the class AllWindowTranslationTest method testFoldWithWindowFunctionEventTime.

@Test
@SuppressWarnings("rawtypes")
public void testFoldWithWindowFunctionEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple2<String, Integer>> window = source.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).fold(new Tuple3<>("", "", 0), new DummyFolder(), new AllWindowFunction<Tuple3<String, String, Integer>, Tuple2<String, Integer>, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Tuple3<String, String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (Tuple3<String, String, Integer> in : values) {
                out.collect(new Tuple2<>(in.f0, in.f2));
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 23 with FoldingStateDescriptor

use of org.apache.flink.api.common.state.FoldingStateDescriptor in project flink by apache.

the class StateBackendTestBase method testFoldingStateDefaultValue.

/**
	 * Verify that an empty {@code FoldingState} yields {@code null}.
	 */
@Test
public void testFoldingStateDefaultValue() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    FoldingStateDescriptor<Integer, String> kvId = new FoldingStateDescriptor<>("id", "Fold-Initial:", new AppendingFold(), String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    FoldingState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(1);
    assertNull(state.get());
    state.add(1);
    state.add(2);
    assertEquals("Fold-Initial:,1,2", state.get());
    state.clear();
    assertNull(state.get());
    backend.dispose();
}

Example 24 with FoldingStateDescriptor

use of org.apache.flink.api.common.state.FoldingStateDescriptor in project flink by apache.

the class WindowTranslationTest method testApplyWithPreFolderEventTime.

@Test
@SuppressWarnings("rawtypes")
public void testApplyWithPreFolderEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple3<String, String, Integer>> window = source.keyBy(new TupleKeySelector()).window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).apply(new Tuple3<>("", "", 0), new DummyFolder(), new WindowFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(String key, TimeWindow window, Iterable<Tuple3<String, String, Integer>> values, Collector<Tuple3<String, String, Integer>> out) throws Exception {
            for (Tuple3<String, String, Integer> in : values) {
                out.collect(new Tuple3<>(in.f0, in.f1, in.f2));
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 25 with FoldingStateDescriptor

use of org.apache.flink.api.common.state.FoldingStateDescriptor in project flink by apache.

the class WindowOperatorTest method testCleanupTimerWithEmptyFoldingStateForTumblingWindows.

@Test
public void testCleanupTimerWithEmptyFoldingStateForTumblingWindows() throws Exception {
    final int WINDOW_SIZE = 2;
    final long LATENESS = 1;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    FoldingStateDescriptor<Tuple2<String, Integer>, Tuple2<String, Integer>> windowStateDesc = new FoldingStateDescriptor<>("window-contents", new Tuple2<>((String) null, 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
            return new Tuple2<>(value.f0, accumulator.f1 + value.f1);
        }
    }, inputType);
    windowStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalSingleValueWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), LATENESS, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    // normal element
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(1599));
    testHarness.processWatermark(new Watermark(1999));
    testHarness.processWatermark(new Watermark(2000));
    testHarness.processWatermark(new Watermark(5000));
    expected.add(new Watermark(1599));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
    // here it fires and purges
    expected.add(new Watermark(1999));
    // here is the cleanup timer
    expected.add(new Watermark(2000));
    expected.add(new Watermark(5000));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.close();
}