Search in sources :

Example 1 with GameActionInfo

use of com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresSpeculative.

/**
 * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
 * on time, and the processing time advances far enough for speculative panes.
 */
@Test
public void testTeamScoresSpeculative() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_TWO, 5, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.BLUE_TWO, 3, Duration.standardSeconds(22))).advanceProcessingTime(Duration.standardMinutes(10)).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(4)), event(TestUser.BLUE_TWO, 2, Duration.standardMinutes(2))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    IntervalWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    // The window contains speculative panes alongside the on-time pane
    PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(blueTeam, 10), /* The on-time blue pane */
    KV.of(redTeam, 9), /* The on-time red pane */
    KV.of(blueTeam, 5), /* The first blue speculative pane */
    KV.of(blueTeam, 8), /* The second blue speculative pane */
    KV.of(redTeam, 5));
    PAssert.that(teamScores).inOnTimePane(window).containsInAnyOrder(KV.of(blueTeam, 10), KV.of(redTeam, 9));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 2 with GameActionInfo

use of com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testUserScore.

/**
 * A test where elements arrive both on-time and late in {@link CalculateUserScores}, which emits
 * output into the {@link GlobalWindow}. All elements that arrive should be taken into account,
 * even if they arrive later than the maximum allowed lateness.
 */
@Test
public void testUserScore() {
    TestStream<GameActionInfo> infos = TestStream.create(AvroCoder.of(GameActionInfo.class)).addElements(event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)).advanceProcessingTime(Duration.standardMinutes(7)).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(5)).advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(Duration.standardHours(12))).addElements(event(TestUser.RED_ONE, 3, Duration.standardMinutes(7)), event(TestUser.RED_ONE, 2, (ALLOWED_LATENESS).plus(Duration.standardHours(13)))).advanceProcessingTime(Duration.standardMinutes(6)).addElements(event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(12))).advanceProcessingTime(Duration.standardMinutes(20)).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> userScores = p.apply(infos).apply(new CalculateUserScores(ALLOWED_LATENESS));
    // User scores are emitted in speculative panes in the Global Window - this matcher choice
    // ensures that panes emitted by the watermark advancing to positive infinity are not included,
    // as that will not occur outside of tests
    PAssert.that(userScores).inEarlyGlobalWindowPanes().containsInAnyOrder(KV.of(TestUser.BLUE_ONE.getUser(), 15), KV.of(TestUser.RED_ONE.getUser(), 7), KV.of(TestUser.RED_ONE.getUser(), 12), KV.of(TestUser.BLUE_TWO.getUser(), 3), KV.of(TestUser.BLUE_TWO.getUser(), 8));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) KV(org.apache.beam.sdk.values.KV) CalculateUserScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateUserScores) Test(org.junit.Test)

Example 3 with GameActionInfo

use of com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class LeaderBoardTest method testTeamScoresOnTime.

/**
 * A test of the {@link CalculateTeamScores} {@link PTransform} when all of the elements arrive
 * on time (ahead of the watermark).
 */
@Test
public void testTeamScoresOnTime() {
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardSeconds(22)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 1, Duration.standardMinutes(4)), event(TestUser.BLUE_ONE, 2, Duration.standardSeconds(270))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    PAssert.that(teamScores).inOnTimePane(new IntervalWindow(baseTime, TEAM_WINDOW_DURATION)).containsInAnyOrder(KV.of(blueTeam, 12), KV.of(redTeam, 4));
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) CalculateTeamScores(com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores) KV(org.apache.beam.sdk.values.KV) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 4 with GameActionInfo

use of com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class HourlyTeamScoreTest method testUserScoresFilter.

/**
 * Test the filtering.
 */
@Test
@Category(ValidatesRunner.class)
public void testUserScoresFilter() throws Exception {
    final Instant startMinTimestamp = new Instant(1447965680000L);
    PCollection<String> input = p.apply(Create.of(GAME_EVENTS).withCoder(StringUtf8Coder.of()));
    PCollection<KV<String, Integer>> output = input.apply("ParseGameEvent", ParDo.of(new ParseEventFn())).apply("FilterStartTime", Filter.by((GameActionInfo gInfo) -> gInfo.getTimestamp() > startMinTimestamp.getMillis())).apply(MapElements.into(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())).via((GameActionInfo gInfo) -> KV.of(gInfo.getUser(), gInfo.getScore())));
    PAssert.that(output).containsInAnyOrder(FILTERED_EVENTS);
    p.run().waitUntilFinish();
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) Instant(org.joda.time.Instant) KV(org.apache.beam.sdk.values.KV) ParseEventFn(com.google.cloud.dataflow.examples.complete.game.UserScore.ParseEventFn) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Example 5 with GameActionInfo

use of com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class UserScoreTest method testParseEventFn.

/**
 * Test the {@link ParseEventFn} {@link org.apache.beam.sdk.transforms.DoFn}.
 */
@Test
public void testParseEventFn() throws Exception {
    DoFnTester<String, GameActionInfo> parseEventFn = DoFnTester.of(new ParseEventFn());
    List<GameActionInfo> results = parseEventFn.processBundle(GAME_EVENTS_ARRAY);
    Assert.assertEquals(results.size(), 8);
    Assert.assertEquals(results.get(0).getUser(), "user0_MagentaKangaroo");
    Assert.assertEquals(results.get(0).getTeam(), "MagentaKangaroo");
    Assert.assertEquals(results.get(0).getScore(), new Integer(3));
}
Also used : GameActionInfo(com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo) ParseEventFn(com.google.cloud.dataflow.examples.complete.game.UserScore.ParseEventFn) Test(org.junit.Test)

Aggregations

GameActionInfo (com.google.cloud.dataflow.examples.complete.game.UserScore.GameActionInfo)9 Test (org.junit.Test)9 KV (org.apache.beam.sdk.values.KV)8 CalculateTeamScores (com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateTeamScores)5 IntervalWindow (org.apache.beam.sdk.transforms.windowing.IntervalWindow)5 ParseEventFn (com.google.cloud.dataflow.examples.complete.game.UserScore.ParseEventFn)3 BoundedWindow (org.apache.beam.sdk.transforms.windowing.BoundedWindow)3 CalculateUserScores (com.google.cloud.dataflow.examples.complete.game.LeaderBoard.CalculateUserScores)2 Instant (org.joda.time.Instant)2 Category (org.junit.experimental.categories.Category)2 ImmutableMap (com.google.common.collect.ImmutableMap)1 Serializable (java.io.Serializable)1 AvroCoder (org.apache.beam.sdk.coders.AvroCoder)1 PipelineOptionsFactory (org.apache.beam.sdk.options.PipelineOptionsFactory)1 PAssert (org.apache.beam.sdk.testing.PAssert)1 TestPipeline (org.apache.beam.sdk.testing.TestPipeline)1 TestStream (org.apache.beam.sdk.testing.TestStream)1 PTransform (org.apache.beam.sdk.transforms.PTransform)1 SerializableFunction (org.apache.beam.sdk.transforms.SerializableFunction)1 GlobalWindow (org.apache.beam.sdk.transforms.windowing.GlobalWindow)1