Search in sources :

Example 1 with CalculateTeamScores

use of com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresSpeculative.

/**
   * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
   * on time, and the processing time advances far enough for speculative panes.
   */
@Test
public void testTeamScoresSpeculative() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_TWO, 5, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.BLUE_TWO, 3, Duration.standardSeconds(22))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(4)), event(TestUser.BLUE_TWO, 2, Duration.standardMinutes(2))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    IntervalWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    // The window contains speculative panes alongside the on-time pane
    PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(blueTeam, 10), /* The on-time blue pane */
    KV.of(redTeam, 9), /* The on-time red pane */
    KV.of(blueTeam, 5), /* The first blue speculative pane */
    KV.of(blueTeam, 8), /* The second blue speculative pane */
    KV.of(redTeam, 5));
    PAssert.that(teamScores).inOnTimePane(window).containsInAnyOrder(KV.of(blueTeam, 10), KV.of(redTeam, 9));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 2 with CalculateTeamScores

use of com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresOnTime.

/**
   * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
   * on time (ahead of the watermark).
   */
@Test
public void testTeamScoresOnTime() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardSeconds(22)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 1, Duration.standardMinutes(4)), event(TestUser.BLUE_ONE, 2, Duration.standardSeconds(270))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    PAssert.that(teamScores).inOnTimePane(new IntervalWindow(baseTime, TEAM_WINDOW_DURATION)).containsInAnyOrder(KV.of(blueTeam, 12), KV.of(redTeam, 4));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 3 with CalculateTeamScores

use of com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresDroppablyLate.

/**
   * A test where elements arrive beyond the maximum allowed lateness. These elements are dropped
   * within {@link CalculateTeamScores} and do not impact the final result.
   */
@Test
public void testTeamScoresDroppablyLate() {
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> infos = TestStream.create(AvroCoder.of(GameActionInfo.class)).addElements(event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)).advanceWatermarkTo(window.maxTimestamp()).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))).addElements(event(TestUser.BLUE_TWO, 3, TEAM_WINDOW_DURATION.minus(Duration.standardSeconds(5))), event(TestUser.RED_ONE, 7, Duration.standardMinutes(4))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(infos).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    // Only one on-time pane and no late panes should be emitted
    PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(redTeam, 7), KV.of(blueTeam, 18));
    // No elements are added before the watermark passes the end of the window plus the allowed
    // lateness, so no refinement should be emitted
    PAssert.that(teamScores).inFinalPane(window).empty();
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 4 with CalculateTeamScores

use of com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresUnobservablyLate.

/**
   * A test where elements arrive behind the watermark (late data), but before the end of the
   * window. These elements are emitted on time.
   */
@Test
public void testTeamScoresUnobservablyLate() {
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))).advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION).minus(Duration.standardMinutes(1))).addElements(event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.BLUE_TWO, 2, Duration.standardSeconds(90)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    // The On Time pane contains the late elements that arrived before the end of the window
    PAssert.that(teamScores).inOnTimePane(window).containsInAnyOrder(KV.of(redTeam, 14), KV.of(blueTeam, 13));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 5 with CalculateTeamScores

use of com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresObservablyLate.

/**
   * A test where elements arrive behind the watermark (late data) after the watermark passes the
   * end of the window, but before the maximum allowed lateness. These elements are emitted in a
   * late pane.
   */
@Test
public void testTeamScoresObservablyLate() {
    Instant firstWindowCloses = baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8))).advanceProcessingTime(Duration.standardMinutes(10)).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 3, Duration.standardMinutes(1)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))).advanceWatermarkTo(firstWindowCloses.minus(Duration.standardMinutes(1))).addElements(event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.RED_TWO, 9, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 1, Duration.standardMinutes(3))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    PAssert.that(teamScores).inWindow(window).satisfies((SerializableFunction<Iterable<KV<String, Integer>>, Void>) input -> {
        assertThat(input, hasItem(KV.of(blueTeam, 11)));
        assertThat(input, hasItem(KV.of(redTeam, 27)));
        return null;
    });
    PAssert.thatMap(teamScores).inOnTimePane(window).isEqualTo(ImmutableMap.<String, Integer>builder().put(redTeam, 7).put(blueTeam, 11).build());
    // No final pane is emitted for the blue team, as all of their updates have been taken into
    // account in earlier panes
    PAssert.that(teamScores).inFinalPane(window).containsInAnyOrder(KV.of(redTeam, 27));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) KV(org.apache.beam.sdk.values.KV) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) SerializableFunction(org.apache.beam.sdk.transforms.SerializableFunction) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) CalculateUserScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateUserScores) PTransform(org.apache.beam.sdk.transforms.PTransform) Assert.assertThat(org.junit.Assert.assertThat) TestPipeline(org.apache.beam.sdk.testing.TestPipeline) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) TimestampedValue(org.apache.beam.sdk.values.TimestampedValue) ImmutableMap(com.google.common.collect.ImmutableMap) PAssert(org.apache.beam.sdk.testing.PAssert) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) AvroCoder(org.apache.beam.sdk.coders.AvroCoder) Serializable(java.io.Serializable) Matchers.hasItem(org.hamcrest.Matchers.hasItem) Rule(org.junit.Rule) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) Instant(org.joda.time.Instant) TestStream(org.apache.beam.sdk.testing.TestStream) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) Instant(org.joda.time.Instant) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) KV(org.apache.beam.sdk.values.KV) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Aggregations

CalculateTeamScores (com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores)5 GameActionInfo (com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo)5 IntervalWindow (org.apache.beam.sdk.transforms.windowing.IntervalWindow)5 KV (org.apache.beam.sdk.values.KV)5 Test (org.junit.Test)5 BoundedWindow (org.apache.beam.sdk.transforms.windowing.BoundedWindow)3 CalculateUserScores (com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateUserScores)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Serializable (java.io.Serializable)1 AvroCoder (org.apache.beam.sdk.coders.AvroCoder)1 PipelineOptionsFactory (org.apache.beam.sdk.options.PipelineOptionsFactory)1 PAssert (org.apache.beam.sdk.testing.PAssert)1 TestPipeline (org.apache.beam.sdk.testing.TestPipeline)1 TestStream (org.apache.beam.sdk.testing.TestStream)1 PTransform (org.apache.beam.sdk.transforms.PTransform)1 SerializableFunction (org.apache.beam.sdk.transforms.SerializableFunction)1 GlobalWindow (org.apache.beam.sdk.transforms.windowing.GlobalWindow)1 PCollection (org.apache.beam.sdk.values.PCollection)1 TimestampedValue (org.apache.beam.sdk.values.TimestampedValue)1 Matchers.hasItem (org.hamcrest.Matchers.hasItem)1