Examples with GlobalWindow org.apache.beam.sdk.transforms.windowing.GlobalWindow used on opensource projects

Example 6 with GlobalWindow

use of org.apache.beam.sdk.transforms.windowing.GlobalWindow in project beam by apache.

the class RepeatedlyStateMachineTest method testRepeatedlyAfterFirstElementCount.

@Test
public void testRepeatedlyAfterFirstElementCount() throws Exception {
    SimpleTriggerStateMachineTester<GlobalWindow> tester = TriggerStateMachineTester.forTrigger(RepeatedlyStateMachine.forever(AfterFirstStateMachine.of(AfterProcessingTimeStateMachine.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(15)), AfterPaneStateMachine.elementCountAtLeast(5))), new GlobalWindows());
    GlobalWindow window = GlobalWindow.INSTANCE;
    tester.injectElements(1);
    assertFalse(tester.shouldFire(window));
    tester.injectElements(2, 3, 4, 5);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.shouldFire(window));
}

Example 7 with GlobalWindow

use of org.apache.beam.sdk.transforms.windowing.GlobalWindow in project beam by apache.

the class ReduceFnRunnerTest method fireNonEmptyOnDrainInGlobalWindow.

/**
 * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to
 * end-of-time.
 */
@Test
public void fireNonEmptyOnDrainInGlobalWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(new GlobalWindows()).withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3))).withMode(AccumulationMode.DISCARDING_FIRED_PANES));
    tester.advanceInputWatermark(new Instant(0));
    final int n = 20;
    for (int i = 0; i < n; i++) {
        tester.injectElements(TimestampedValue.of(i, new Instant(i)));
    }
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertEquals(n / 3, output.size());
    for (int i = 0; i < output.size(); i++) {
        assertEquals(Timing.EARLY, output.get(i).getPane().getTiming());
        assertEquals(i, output.get(i).getPane().getIndex());
        assertEquals(3, Iterables.size(output.get(i).getValue()));
    }
    tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    output = tester.extractOutput();
    assertEquals(1, output.size());
    assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming());
    assertEquals(n / 3, output.get(0).getPane().getIndex());
    assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue()));
}

Example 8 with GlobalWindow

use of org.apache.beam.sdk.transforms.windowing.GlobalWindow in project beam by apache.

the class DoFnOperatorTest method testStateRestore.

@Test
public void testStateRestore() throws Exception {
    DoFn<KV<String, Long>, KV<String, Long>> filterElementsEqualToCountFn = new DoFn<KV<String, Long>, KV<String, Long>>() {

        @StateId("counter")
        private final StateSpec<ValueState<Long>> counterSpec = StateSpecs.value(VarLongCoder.of());

        @ProcessElement
        public void processElement(ProcessContext context, @StateId("counter") ValueState<Long> count) {
            long currentCount = Optional.ofNullable(count.read()).orElse(0L);
            currentCount = currentCount + 1;
            count.write(currentCount);
            KV<String, Long> currentElement = context.element();
            if (currentCount == currentElement.getValue()) {
                context.output(currentElement);
            }
        }
    };
    WindowingStrategy<Object, GlobalWindow> windowingStrategy = WindowingStrategy.globalDefault();
    TupleTag<KV<String, Long>> outputTag = new TupleTag<>("main-output");
    StringUtf8Coder keyCoder = StringUtf8Coder.of();
    KvToByteBufferKeySelector<String, Long> keySelector = new KvToByteBufferKeySelector<>(keyCoder, null);
    KvCoder<String, Long> coder = KvCoder.of(keyCoder, VarLongCoder.of());
    FullWindowedValueCoder<KV<String, Long>> kvCoder = WindowedValue.getFullCoder(coder, windowingStrategy.getWindowFn().windowCoder());
    CoderTypeInformation<ByteBuffer> keyCoderInfo = new CoderTypeInformation<>(FlinkKeyUtils.ByteBufferCoder.of(), FlinkPipelineOptions.defaults());
    OneInputStreamOperatorTestHarness<WindowedValue<KV<String, Long>>, WindowedValue<KV<String, Long>>> testHarness = createTestHarness(windowingStrategy, filterElementsEqualToCountFn, kvCoder, kvCoder, keyCoder, outputTag, keyCoderInfo, keySelector);
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 100L))));
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 100L))));
    OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    testHarness = createTestHarness(windowingStrategy, filterElementsEqualToCountFn, kvCoder, kvCoder, keyCoder, outputTag, keyCoderInfo, keySelector);
    testHarness.initializeState(snapshot);
    testHarness.open();
    // after restore: counter = 2
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 100L))));
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 4L))));
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 5L))));
    testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow(KV.of("a", 100L))));
    assertThat(stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains(WindowedValue.valueInGlobalWindow(KV.of("a", 4L)), WindowedValue.valueInGlobalWindow(KV.of("a", 5L))));
    testHarness.close();
}

Example 9 with GlobalWindow

use of org.apache.beam.sdk.transforms.windowing.GlobalWindow in project beam by apache.

the class DoFnOperatorTest method testGCForGlobalWindow.

@Test
public void testGCForGlobalWindow() throws Exception {
    WindowingStrategy<Object, GlobalWindow> windowingStrategy = WindowingStrategy.globalDefault();
    KeyedOneInputStreamOperatorTestHarness<ByteBuffer, WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness = getHarness(windowingStrategy, 5000, (window) -> new Instant(50), 4092);
    testHarness.open();
    testHarness.processWatermark(0);
    // ensure the state was garbage collected and the pending timers have been removed
    assertEquals(0, testHarness.numKeyedStateEntries());
    // Check global window cleanup via final watermark, _without_ cleanup timers
    testHarness.processElement(new StreamRecord<>(WindowedValue.of(KV.of("key1", 5), new Instant(23), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)));
    testHarness.processElement(new StreamRecord<>(WindowedValue.of(KV.of("key2", 6), new Instant(42), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)));
    // timers set by the transform
    assertThat(testHarness.numEventTimeTimers(), is(2));
    // state has been written which needs to be cleaned up (includes timers)
    assertThat(testHarness.numKeyedStateEntries(), is(4));
    // Fire timers set
    testHarness.processWatermark(51);
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(testHarness.numKeyedStateEntries(), is(2));
    // Should not trigger garbage collection yet
    testHarness.processWatermark(GlobalWindow.INSTANCE.maxTimestamp().plus(Duration.millis(1)).getMillis());
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(testHarness.numKeyedStateEntries(), is(2));
    // Cleanup due to end of global window
    testHarness.processWatermark(GlobalWindow.INSTANCE.maxTimestamp().plus(Duration.millis(2)).getMillis());
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(testHarness.numKeyedStateEntries(), is(0));
    // Any new state will also be cleaned up on close
    testHarness.processElement(new StreamRecord<>(WindowedValue.of(KV.of("key2", 6), new Instant(42), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)));
    // Close sends Flink's max watermark and will cleanup again
    testHarness.close();
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(testHarness.numKeyedStateEntries(), is(0));
}

Example 10 with GlobalWindow

use of org.apache.beam.sdk.transforms.windowing.GlobalWindow in project beam by apache.

the class ProcessFnRunner method checkTrivialOuterWindows.

private static <T> void checkTrivialOuterWindows(WindowedValue<KeyedWorkItem<byte[], T>> windowedKWI) {
    // In practice it will be in 0 or 1 windows (ValueInEmptyWindows or ValueInGlobalWindow)
    Collection<? extends BoundedWindow> outerWindows = windowedKWI.getWindows();
    if (!outerWindows.isEmpty()) {
        checkArgument(outerWindows.size() == 1, "The KeyedWorkItem itself must not be in multiple windows, but was in: %s", outerWindows);
        BoundedWindow onlyWindow = Iterables.getOnlyElement(outerWindows);
        checkArgument(onlyWindow instanceof GlobalWindow, "KeyedWorkItem must be in the Global window, but was in: %s", onlyWindow);
    }
}