Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 11 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresSpeculative.

/**
 * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
 * on time, and the processing time advances far enough for speculative panes.
 */
@Test
public void testTeamScoresSpeculative() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_TWO, 5, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.BLUE_TWO, 3, Duration.standardSeconds(22))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(4)), event(TestUser.BLUE_TWO, 2, Duration.standardMinutes(2))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    IntervalWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    // The window contains speculative panes alongside the on-time pane
    PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(blueTeam, 10), /* The on-time blue pane */
    KV.of(redTeam, 9), /* The on-time red pane */
    KV.of(blueTeam, 5), /* The first blue speculative pane */
    KV.of(blueTeam, 8), /* The second blue speculative pane */
    KV.of(redTeam, 5));
    PAssert.that(teamScores).inOnTimePane(window).containsInAnyOrder(KV.of(blueTeam, 10), KV.of(redTeam, 9));
    p.run().waitUntilFinish();
}

Example 12 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresOnTime.

/**
 * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
 * on time (ahead of the watermark).
 */
@Test
public void testTeamScoresOnTime() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardSeconds(22)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 1, Duration.standardMinutes(4)), event(TestUser.BLUE_ONE, 2, Duration.standardSeconds(270))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    PAssert.that(teamScores).inOnTimePane(new IntervalWindow(baseTime, TEAM_WINDOW_DURATION)).containsInAnyOrder(KV.of(blueTeam, 12), KV.of(redTeam, 4));
    p.run().waitUntilFinish();
}

Example 13 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class HourlyTeamScore method configureOutput.

/**
 * Create a map of information that describes how to write pipeline output to text. This map
 * is passed to the {@link WriteToText} constructor to write team score sums and
 * includes information about window start time.
 */
protected static Map<String, WriteToText.FieldFn<KV<String, Integer>>> configureOutput() {
    Map<String, WriteToText.FieldFn<KV<String, Integer>>> config = new HashMap<String, WriteToText.FieldFn<KV<String, Integer>>>();
    config.put("team", (c, w) -> c.element().getKey());
    config.put("total_score", (c, w) -> c.element().getValue());
    config.put("window_start", (c, w) -> {
        IntervalWindow window = (IntervalWindow) w;
        return fmt.print(window.start());
    });
    return config;
}

Example 14 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class TimestampPrefixingWindowCoderTest method testEncodeAndDecode.

@Test
public void testEncodeAndDecode() throws Exception {
    List<IntervalWindow> intervalWindowsToTest = Lists.newArrayList(new IntervalWindow(new Instant(0L), new Instant(1L)), new IntervalWindow(new Instant(100L), new Instant(200L)), new IntervalWindow(new Instant(0L), BoundedWindow.TIMESTAMP_MAX_VALUE));
    TimestampPrefixingWindowCoder<IntervalWindow> coder1 = TimestampPrefixingWindowCoder.of(IntervalWindow.getCoder());
    for (IntervalWindow window : intervalWindowsToTest) {
        CoderProperties.coderDecodeEncodeEqual(coder1, window);
    }
    GlobalWindow globalWindow = GlobalWindow.INSTANCE;
    TimestampPrefixingWindowCoder<GlobalWindow> coder2 = TimestampPrefixingWindowCoder.of(GlobalWindow.Coder.INSTANCE);
    CoderProperties.coderDecodeEncodeEqual(coder2, globalWindow);
    TimestampPrefixingWindowCoder<CustomWindow> coder3 = TimestampPrefixingWindowCoder.of(CustomWindowCoder.of(true, true));
    for (CustomWindow window : CUSTOM_WINDOW_LIST) {
        CoderProperties.coderDecodeEncodeEqual(coder3, window);
    }
}

Example 15 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class AssignWindowsRunnerTest method factoryCreatesFromJavaWindowFn.

@Test
public void factoryCreatesFromJavaWindowFn() throws Exception {
    SdkComponents components = SdkComponents.create();
    components.registerEnvironment(Environments.createDockerEnvironment("java"));
    PTransform windowPTransform = PTransform.newBuilder().putInputs("in", "input").putOutputs("out", "output").setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN).setPayload(WindowIntoPayload.newBuilder().setWindowFn(WindowingStrategyTranslation.toProto(new TestWindowFn(), components)).build().toByteString()).build()).build();
    ThrowingFunction<WindowedValue<?>, WindowedValue<?>> fn = (ThrowingFunction) factory.forPTransform("transform", windowPTransform);
    assertThat(fn.apply(WindowedValue.of(22L, new Instant(5), new IntervalWindow(new Instant(0L), new Instant(20027L)), PaneInfo.ON_TIME_AND_ONLY_FIRING)), equalTo(WindowedValue.of(22L, new Instant(5), new TestWindowFn().assignWindow(new Instant(5)), PaneInfo.ON_TIME_AND_ONLY_FIRING)));
}