Examples with Trigger org.apache.flink.streaming.api.windowing.triggers.Trigger used on opensource projects

Example 1 with Trigger

use of org.apache.flink.streaming.api.windowing.triggers.Trigger in project flink by apache.

the class WindowOperatorContractTest method testWindowsAreMergedEagerly.

/**
 * Verify that windows are merged eagerly, if possible.
 */
private void testWindowsAreMergedEagerly(final TimeDomainAdaptor timeAdaptor) throws Exception {
    // in this test we only have one state window and windows are eagerly
    // merged into the first window
    MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
    timeAdaptor.setIsEventTime(mockAssigner);
    Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
    InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);
    testHarness.open();
    timeAdaptor.advanceTime(testHarness, Long.MIN_VALUE);
    assertEquals(0, testHarness.extractOutputStreamRecords().size());
    assertEquals(0, testHarness.numKeyedStateEntries());
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3];
            // don't interfere with cleanup timers
            timeAdaptor.registerTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).update("hello");
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.OnMergeContext context = (Trigger.OnMergeContext) invocation.getArguments()[1];
            // don't interfere with cleanup timers
            timeAdaptor.registerTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).update("hello");
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onMerge(anyTimeWindow(), anyOnMergeContext());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[1];
            timeAdaptor.deleteTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).clear();
            return null;
        }
    }).when(mockTrigger).clear(anyTimeWindow(), anyTriggerContext());
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(0, 2)));
    testHarness.processElement(new StreamRecord<>(0, 0L));
    assertEquals(3, // window state plus trigger state plus merging
    testHarness.numKeyedStateEntries());
    // window set
    // timer and GC timer
    assertEquals(2, timeAdaptor.numTimers(testHarness));
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(2, 4)));
    shouldMergeWindows(mockAssigner, new ArrayList<>(Arrays.asList(new TimeWindow(0, 2), new TimeWindow(2, 4))), new ArrayList<>(Arrays.asList(new TimeWindow(0, 2), new TimeWindow(2, 4))), new TimeWindow(0, 4));
    // don't register a timer or update state in onElement, this checks
    // whether onMerge does correctly set those things
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    testHarness.processElement(new StreamRecord<>(0, 0L));
    verify(mockTrigger).onMerge(eq(new TimeWindow(0, 4)), anyOnMergeContext());
    assertEquals(3, testHarness.numKeyedStateEntries());
    assertEquals(2, timeAdaptor.numTimers(testHarness));
}

Example 2 with Trigger

use of org.apache.flink.streaming.api.windowing.triggers.Trigger in project flink by apache.

the class WindowOperatorContractTest method testStateTypeIsConsistentFromWindowStateAndGlobalState.

@Test
public void testStateTypeIsConsistentFromWindowStateAndGlobalState() throws Exception {
    class NoOpAggregateFunction implements AggregateFunction<String, String, String> {

        @Override
        public String createAccumulator() {
            return null;
        }

        @Override
        public String add(String value, String accumulator) {
            return null;
        }

        @Override
        public String getResult(String accumulator) {
            return null;
        }

        @Override
        public String merge(String a, String b) {
            return null;
        }
    }
    WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
    Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
    InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);
    testHarness.open();
    when(mockTrigger.onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE);
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(0, 20)));
    AtomicBoolean processWasInvoked = new AtomicBoolean(false);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext) invocationOnMock.getArguments()[2];
            KeyedStateStore windowKeyedStateStore = context.windowState();
            KeyedStateStore globalKeyedStateStore = context.globalState();
            ListStateDescriptor<String> windowListStateDescriptor = new ListStateDescriptor<String>("windowListState", String.class);
            ListStateDescriptor<String> globalListStateDescriptor = new ListStateDescriptor<String>("globalListState", String.class);
            assertEquals(windowKeyedStateStore.getListState(windowListStateDescriptor).getClass(), globalKeyedStateStore.getListState(globalListStateDescriptor).getClass());
            ValueStateDescriptor<String> windowValueStateDescriptor = new ValueStateDescriptor<String>("windowValueState", String.class);
            ValueStateDescriptor<String> globalValueStateDescriptor = new ValueStateDescriptor<String>("globalValueState", String.class);
            assertEquals(windowKeyedStateStore.getState(windowValueStateDescriptor).getClass(), globalKeyedStateStore.getState(globalValueStateDescriptor).getClass());
            AggregatingStateDescriptor<String, String, String> windowAggStateDesc = new AggregatingStateDescriptor<String, String, String>("windowAgg", new NoOpAggregateFunction(), String.class);
            AggregatingStateDescriptor<String, String, String> globalAggStateDesc = new AggregatingStateDescriptor<String, String, String>("globalAgg", new NoOpAggregateFunction(), String.class);
            assertEquals(windowKeyedStateStore.getAggregatingState(windowAggStateDesc).getClass(), globalKeyedStateStore.getAggregatingState(globalAggStateDesc).getClass());
            ReducingStateDescriptor<String> windowReducingStateDesc = new ReducingStateDescriptor<String>("windowReducing", (a, b) -> a, String.class);
            ReducingStateDescriptor<String> globalReducingStateDesc = new ReducingStateDescriptor<String>("globalReducing", (a, b) -> a, String.class);
            assertEquals(windowKeyedStateStore.getReducingState(windowReducingStateDesc).getClass(), globalKeyedStateStore.getReducingState(globalReducingStateDesc).getClass());
            MapStateDescriptor<String, String> windowMapStateDescriptor = new MapStateDescriptor<String, String>("windowMapState", String.class, String.class);
            MapStateDescriptor<String, String> globalMapStateDescriptor = new MapStateDescriptor<String, String>("globalMapState", String.class, String.class);
            assertEquals(windowKeyedStateStore.getMapState(windowMapStateDescriptor).getClass(), globalKeyedStateStore.getMapState(globalMapStateDescriptor).getClass());
            processWasInvoked.set(true);
            return null;
        }
    }).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());
    testHarness.processElement(new StreamRecord<>(0, 0L));
    assertTrue(processWasInvoked.get());
}

Example 3 with Trigger

use of org.apache.flink.streaming.api.windowing.triggers.Trigger in project flink by apache.

the class WindowOperatorContractTest method testStateSnapshotAndRestore.

@Test
public void testStateSnapshotAndRestore() throws Exception {
    MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
    Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
    InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);
    testHarness.open();
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));
    assertEquals(0, testHarness.getOutput().size());
    assertEquals(0, testHarness.numKeyedStateEntries());
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3];
            context.registerEventTimeTimer(0L);
            context.getPartitionedState(valueStateDescriptor).update("hello");
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    shouldFireAndPurgeOnEventTime(mockTrigger);
    testHarness.processElement(new StreamRecord<>(0, 0L));
    // window-contents and trigger state for two windows plus merging window set
    assertEquals(5, testHarness.numKeyedStateEntries());
    // timers/gc timers for two windows
    assertEquals(4, testHarness.numEventTimeTimers());
    OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
    // restore
    mockAssigner = mockMergingAssigner();
    mockTrigger = mockTrigger();
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[1];
            context.deleteEventTimeTimer(0L);
            context.getPartitionedState(valueStateDescriptor).clear();
            return null;
        }
    }).when(mockTrigger).clear(anyTimeWindow(), anyTriggerContext());
    // only fire on the timestamp==0L timers, not the gc timers
    when(mockTrigger.onEventTime(eq(0L), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE);
    mockWindowFunction = mockWindowFunction();
    testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    assertEquals(0, testHarness.extractOutputStreamRecords().size());
    // verify that we still have all the state/timers/merging window set
    assertEquals(5, testHarness.numKeyedStateEntries());
    // timers/gc timers for two windows
    assertEquals(4, testHarness.numEventTimeTimers());
    verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());
    testHarness.processWatermark(new Watermark(20L));
    verify(mockTrigger, times(2)).clear(anyTimeWindow(), anyTriggerContext());
    verify(mockWindowFunction, times(2)).process(eq(0), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());
    verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());
    verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(2, 4)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());
    // it's also called for the cleanup timers
    verify(mockTrigger, times(4)).onEventTime(anyLong(), anyTimeWindow(), anyTriggerContext());
    verify(mockTrigger, times(1)).onEventTime(eq(0L), eq(new TimeWindow(0, 2)), anyTriggerContext());
    verify(mockTrigger, times(1)).onEventTime(eq(0L), eq(new TimeWindow(2, 4)), anyTriggerContext());
    assertEquals(0, testHarness.numKeyedStateEntries());
    assertEquals(0, testHarness.numEventTimeTimers());
}

Example 4 with Trigger

use of org.apache.flink.streaming.api.windowing.triggers.Trigger in project flink by apache.

the class WindowOperatorContractTest method testMergingOfExistingWindows.

/**
 * Verify that we only keep one of the underlying state windows. This test also verifies that GC
 * timers are correctly deleted when merging windows.
 */
private void testMergingOfExistingWindows(final TimeDomainAdaptor timeAdaptor) throws Exception {
    MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
    timeAdaptor.setIsEventTime(mockAssigner);
    Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
    InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);
    testHarness.open();
    timeAdaptor.advanceTime(testHarness, Long.MIN_VALUE);
    assertEquals(0, testHarness.extractOutputStreamRecords().size());
    assertEquals(0, testHarness.numKeyedStateEntries());
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3];
            // don't interfere with cleanup timers
            timeAdaptor.registerTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).update("hello");
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.OnMergeContext context = (Trigger.OnMergeContext) invocation.getArguments()[1];
            // don't interfere with cleanup timers
            timeAdaptor.registerTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).update("hello");
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onMerge(anyTimeWindow(), anyOnMergeContext());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[1];
            // don't interfere with cleanup timers
            timeAdaptor.deleteTimer(context, 0L);
            context.getPartitionedState(valueStateDescriptor).clear();
            return null;
        }
    }).when(mockTrigger).clear(anyTimeWindow(), anyTriggerContext());
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(0, 2)));
    testHarness.processElement(new StreamRecord<>(0, 0L));
    assertEquals(3, // window state plus trigger state plus merging
    testHarness.numKeyedStateEntries());
    // window set
    // trigger timer plus GC timer
    assertEquals(2, timeAdaptor.numTimers(testHarness));
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(2, 4)));
    testHarness.processElement(new StreamRecord<>(0, 0L));
    assertEquals(5, // window state plus trigger state plus merging
    testHarness.numKeyedStateEntries());
    // window set
    // trigger timer plus GC timer
    assertEquals(4, timeAdaptor.numTimers(testHarness));
    when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())).thenReturn(Arrays.asList(new TimeWindow(1, 3)));
    shouldMergeWindows(mockAssigner, new ArrayList<>(Arrays.asList(new TimeWindow(0, 2), new TimeWindow(2, 4), new TimeWindow(1, 3))), new ArrayList<>(Arrays.asList(new TimeWindow(0, 2), new TimeWindow(2, 4), new TimeWindow(1, 3))), new TimeWindow(0, 4));
    testHarness.processElement(new StreamRecord<>(0, 0L));
    assertEquals(3, testHarness.numKeyedStateEntries());
    // window set
    // trigger timer plus GC timer
    assertEquals(2, timeAdaptor.numTimers(testHarness));
    assertEquals(0, testHarness.extractOutputStreamRecords().size());
}

Example 5 with Trigger

use of org.apache.flink.streaming.api.windowing.triggers.Trigger in project flink by apache.

the class PurgingTriggerTest method testForwarding.

@Test
public void testForwarding() throws Exception {
    Trigger<Object, TimeWindow> mockTrigger = mock(Trigger.class);
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(PurgingTrigger.of(mockTrigger), new TimeWindow.Serializer());
    when(mockTrigger.onElement(Matchers.anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.CONTINUE);
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    when(mockTrigger.onElement(Matchers.anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    when(mockTrigger.onElement(Matchers.anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE_AND_PURGE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    when(mockTrigger.onElement(Matchers.anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.PURGE);
    assertEquals(TriggerResult.PURGE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3];
            // register some timers that we can step through to call onEventTime
            // several
            // times in a row
            context.registerEventTimeTimer(1);
            context.registerEventTimeTimer(2);
            context.registerEventTimeTimer(3);
            context.registerEventTimeTimer(4);
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    // set up our timers
    testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2));
    assertEquals(4, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    when(mockTrigger.onEventTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.CONTINUE);
    assertEquals(TriggerResult.CONTINUE, testHarness.advanceWatermark(1, new TimeWindow(0, 2)));
    when(mockTrigger.onEventTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.advanceWatermark(2, new TimeWindow(0, 2)));
    when(mockTrigger.onEventTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE_AND_PURGE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.advanceWatermark(3, new TimeWindow(0, 2)));
    when(mockTrigger.onEventTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.PURGE);
    assertEquals(TriggerResult.PURGE, testHarness.advanceWatermark(4, new TimeWindow(0, 2)));
    doAnswer(new Answer<TriggerResult>() {

        @Override
        public TriggerResult answer(InvocationOnMock invocation) throws Exception {
            Trigger.TriggerContext context = (Trigger.TriggerContext) invocation.getArguments()[3];
            // register some timers that we can step through to call onEventTime
            // several
            // times in a row
            context.registerProcessingTimeTimer(1);
            context.registerProcessingTimeTimer(2);
            context.registerProcessingTimeTimer(3);
            context.registerProcessingTimeTimer(4);
            return TriggerResult.CONTINUE;
        }
    }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());
    // set up our timers
    testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2));
    assertEquals(4, testHarness.numProcessingTimeTimers(new TimeWindow(0, 2)));
    assertEquals(0, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    when(mockTrigger.onProcessingTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.CONTINUE);
    assertEquals(TriggerResult.CONTINUE, testHarness.advanceProcessingTime(1, new TimeWindow(0, 2)));
    when(mockTrigger.onProcessingTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.advanceProcessingTime(2, new TimeWindow(0, 2)));
    when(mockTrigger.onProcessingTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.FIRE_AND_PURGE);
    assertEquals(TriggerResult.FIRE_AND_PURGE, testHarness.advanceProcessingTime(3, new TimeWindow(0, 2)));
    when(mockTrigger.onProcessingTime(anyLong(), anyTimeWindow(), anyTriggerContext())).thenReturn(TriggerResult.PURGE);
    assertEquals(TriggerResult.PURGE, testHarness.advanceProcessingTime(4, new TimeWindow(0, 2)));
    testHarness.mergeWindows(new TimeWindow(0, 2), Collections.singletonList(new TimeWindow(0, 1)));
    verify(mockTrigger, times(1)).onMerge(anyTimeWindow(), anyOnMergeContext());
    testHarness.clearTriggerState(new TimeWindow(0, 2));
    verify(mockTrigger, times(1)).clear(eq(new TimeWindow(0, 2)), anyTriggerContext());
}