Examples with WindowedValue org.apache.beam.sdk.util.WindowedValue used on opensource projects

Example 86 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testMergingWithReusedWindow.

/**
   * If a later event tries to reuse an earlier session window which has been closed, we
   * should reject that element and not fail due to the window no longer being active.
   */
@Test
public void testMergingWithReusedWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // One elements in one session window.
    // in [1, 11), gc at 21.
    tester.injectElements(TimestampedValue.of(1, new Instant(1)));
    // Close the trigger, but the gargbage collection timer is still pending.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    tester.advanceInputWatermark(new Instant(15));
    // Another element in the same session window.
    // Should be discarded with 'window closed'.
    // in [1, 11), gc at 21.
    tester.injectElements(TimestampedValue.of(1, new Instant(1)));
    // And nothing should be left in the active window state.
    assertTrue(tester.hasNoActiveWindows());
    // Now the garbage collection timer will fire, finding the trigger already closed.
    tester.advanceInputWatermark(new Instant(100));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1), // timestamp
    1, // window start
    1, // window end
    11));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)));
}

Example 87 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method noEmptyPanesFinalAlways.

@Test
public void noEmptyPanesFinalAlways() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.<IntervalWindow>forever(AfterFirst.<IntervalWindow>of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    tester.advanceInputWatermark(new Instant(20));
    tester.advanceInputWatermark(new Instant(250));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(// Trigger with 2 elements
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10), // Trigger for the final pane
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}

Example 88 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testMergeBeforeFinalizing.

@Test
public void testMergeBeforeFinalizing() throws Exception {
    // Verify that we merge windows before producing output so users don't see undesired
    // unmerged windows.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(0), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // All on time data, verify watermark hold.
    // These two windows should pre-merge immediately to [1, 20)
    tester.injectElements(// in [1, 11)
    TimestampedValue.of(1, new Instant(1)), // in [10, 20)
    TimestampedValue.of(10, new Instant(10)));
    // And this should fire the end-of-window timer
    tester.advanceInputWatermark(new Instant(100));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 10), // timestamp
    1, // window start
    1, // window end
    20));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)));
}

Example 89 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class BatchViewOverridesTest method testToIsmMetadataRecordForKeyDoFn.

@Test
public void testToIsmMetadataRecordForKeyDoFn() throws Exception {
    TupleTag<KV<Integer, KV<IntervalWindow, Long>>> outputForSizeTag = new TupleTag<>();
    TupleTag<KV<Integer, KV<IntervalWindow, Long>>> outputForEntrySetTag = new TupleTag<>();
    Coder<Long> keyCoder = VarLongCoder.of();
    Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
    IsmRecordCoder<WindowedValue<Long>> ismCoder = IsmRecordCoder.of(1, 2, ImmutableList.<Coder<?>>of(MetadataKeyCoder.of(keyCoder), IntervalWindow.getCoder(), BigEndianLongCoder.of()), FullWindowedValueCoder.of(VarLongCoder.of(), windowCoder));
    DoFnTester<KV<Integer, Iterable<KV<IntervalWindow, Long>>>, IsmRecord<WindowedValue<Long>>> doFnTester = DoFnTester.of(new BatchViewOverrides.BatchViewAsMultimap.ToIsmMetadataRecordForKeyDoFn<Long, Long, IntervalWindow>(keyCoder, windowCoder));
    IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
    IntervalWindow windowB = new IntervalWindow(new Instant(10), new Instant(20));
    IntervalWindow windowC = new IntervalWindow(new Instant(20), new Instant(30));
    Iterable<KV<Integer, Iterable<KV<IntervalWindow, Long>>>> inputElements = ImmutableList.of(KV.of(1, (Iterable<KV<IntervalWindow, Long>>) ImmutableList.of(KV.of(windowA, 2L), // same window as previous
    KV.of(windowA, 3L), // different window as previous
    KV.of(windowB, 3L))), KV.of(ismCoder.hash(ImmutableList.of(IsmFormat.getMetadataKey(), windowB)), (Iterable<KV<IntervalWindow, Long>>) ImmutableList.of(KV.of(windowC, 3L))));
    // The order of the output elements is important relative to processing order
    assertThat(doFnTester.processBundle(inputElements), contains(IsmRecord.<WindowedValue<Long>>meta(ImmutableList.of(IsmFormat.getMetadataKey(), windowA, 1L), CoderUtils.encodeToByteArray(VarLongCoder.of(), 2L)), IsmRecord.<WindowedValue<Long>>meta(ImmutableList.of(IsmFormat.getMetadataKey(), windowA, 2L), CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L)), IsmRecord.<WindowedValue<Long>>meta(ImmutableList.of(IsmFormat.getMetadataKey(), windowB, 1L), CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L)), IsmRecord.<WindowedValue<Long>>meta(ImmutableList.of(IsmFormat.getMetadataKey(), windowC, 1L), CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L))));
}