Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 21 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class FailsafeValueInSingleWindowCoderTest method testDecodeEncodeEqual.

@Test
public void testDecodeEncodeEqual() throws Exception {
    Instant now = Instant.now();
    FailsafeValueInSingleWindow<String, String> value = FailsafeValueInSingleWindow.of("foo", now, new IntervalWindow(now, now.plus(Duration.standardSeconds(10))), PaneInfo.NO_FIRING, "bar");
    CoderProperties.coderDecodeEncodeEqual(FailsafeValueInSingleWindow.Coder.of(StringUtf8Coder.of(), StringUtf8Coder.of(), IntervalWindow.getCoder()), value);
}

Example 22 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class BigtableIOTest method testWritingAndWaitingOnResults.

/**
 * Tests that the outputs of the Bigtable writer are correctly windowed, and can be used in a
 * Wait.on transform as the trigger.
 */
@Test
public void testWritingAndWaitingOnResults() throws Exception {
    final String table = "table";
    final String key = "key";
    final String value = "value";
    service.createTable(table);
    Instant elementTimestamp = Instant.parse("2019-06-10T00:00:00");
    Duration windowDuration = Duration.standardMinutes(1);
    TestStream<KV<ByteString, Iterable<Mutation>>> writeInputs = TestStream.create(bigtableCoder).advanceWatermarkTo(elementTimestamp).addElements(makeWrite(key, value)).advanceWatermarkToInfinity();
    TestStream<String> testInputs = TestStream.create(StringUtf8Coder.of()).advanceWatermarkTo(elementTimestamp).addElements("done").advanceWatermarkToInfinity();
    PCollection<BigtableWriteResult> writes = p.apply("rows", writeInputs).apply("window rows", Window.<KV<ByteString, Iterable<Mutation>>>into(FixedWindows.of(windowDuration)).withAllowedLateness(Duration.ZERO)).apply("write", defaultWrite.withTableId(table).withWriteResults());
    PCollection<String> inputs = p.apply("inputs", testInputs).apply("window inputs", Window.into(FixedWindows.of(windowDuration))).apply("wait", Wait.on(writes));
    BoundedWindow expectedWindow = new IntervalWindow(elementTimestamp, windowDuration);
    PAssert.that(inputs).inWindow(expectedWindow).containsInAnyOrder("done");
    p.run();
}

Example 23 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class TestStreamTest method testEarlyPanesOfWindow.

@Test
@Category({ ValidatesRunner.class, UsesTestStreamWithProcessingTime.class })
public void testEarlyPanesOfWindow() {
    TestStream<Long> source = TestStream.create(VarLongCoder.of()).addElements(TimestampedValue.of(1L, new Instant(1000L))).advanceProcessingTime(// Fire early pane
    Duration.standardMinutes(6)).addElements(TimestampedValue.of(2L, new Instant(2000L))).advanceProcessingTime(// Fire early pane
    Duration.standardMinutes(6)).addElements(TimestampedValue.of(3L, new Instant(3000L))).advanceProcessingTime(// Fire early pane
    Duration.standardMinutes(6)).advanceWatermarkToInfinity();
    PCollection<KV<String, Long>> sum = p.apply(source).apply(Window.<Long>into(FixedWindows.of(Duration.standardMinutes(30))).triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(5)))).accumulatingFiredPanes().withAllowedLateness(Duration.ZERO)).apply(MapElements.into(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs())).via(v -> KV.of("key", v))).apply(Sum.longsPerKey());
    IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(30)));
    PAssert.that(sum).inEarlyPane(window).satisfies(input -> {
        assertThat(StreamSupport.stream(input.spliterator(), false).count(), is(3L));
        return null;
    }).containsInAnyOrder(KV.of("key", 1L), KV.of("key", 3L), KV.of("key", 6L)).inOnTimePane(window).satisfies(input -> {
        assertThat(StreamSupport.stream(input.spliterator(), false).count(), is(1L));
        return null;
    }).containsInAnyOrder(KV.of("key", 6L));
    p.run().waitUntilFinish();
}

Example 24 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ValueInSingleWindowCoderTest method testDecodeEncodeEqual.

@Test
public void testDecodeEncodeEqual() throws Exception {
    Instant now = Instant.now();
    ValueInSingleWindow<String> value = ValueInSingleWindow.of("foo", now, new IntervalWindow(now, now.plus(Duration.standardSeconds(10))), PaneInfo.NO_FIRING);
    CoderProperties.coderDecodeEncodeEqual(ValueInSingleWindow.Coder.of(StringUtf8Coder.of(), IntervalWindow.getCoder()), value);
}

Example 25 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class DoFnTesterTest method testSupportsWindowParameter.

@Test
public void testSupportsWindowParameter() throws Exception {
    Instant now = Instant.now();
    try (DoFnTester<Integer, KV<Integer, BoundedWindow>> tester = DoFnTester.of(new DoFnWithWindowParameter())) {
        BoundedWindow firstWindow = new IntervalWindow(now, now.plus(Duration.standardMinutes(1)));
        tester.processWindowedElement(1, now, firstWindow);
        tester.processWindowedElement(2, now, firstWindow);
        BoundedWindow secondWindow = new IntervalWindow(now, now.plus(Duration.standardMinutes(4)));
        tester.processWindowedElement(3, now, secondWindow);
        tester.finishBundle();
        assertThat(tester.peekOutputElementsInWindow(firstWindow), containsInAnyOrder(TimestampedValue.of(KV.of(1, firstWindow), now), TimestampedValue.of(KV.of(2, firstWindow), now)));
        assertThat(tester.peekOutputElementsInWindow(secondWindow), containsInAnyOrder(TimestampedValue.of(KV.of(3, secondWindow), now)));
    }
}