use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.
the class LeaderBoardTest method testTeamScoresObservablyLate.
/**
* A test where elements arrive behind the watermark (late data) after the watermark passes the
* end of the window, but before the maximum allowed lateness. These elements are emitted in a
* late pane.
*/
@Test
public void testTeamScoresObservablyLate() {
Instant firstWindowCloses = baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION);
TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8))).advanceProcessingTime(Duration.standardMinutes(10)).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 3, Duration.standardMinutes(1)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))).advanceWatermarkTo(firstWindowCloses.minus(Duration.standardMinutes(1))).addElements(event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.RED_TWO, 9, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 1, Duration.standardMinutes(3))).advanceWatermarkToInfinity();
PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
String blueTeam = TestUser.BLUE_ONE.getTeam();
String redTeam = TestUser.RED_ONE.getTeam();
PAssert.that(teamScores).inWindow(window).satisfies((SerializableFunction<Iterable<KV<String, Integer>>, Void>) input -> {
assertThat(input, hasItem(KV.of(blueTeam, 11)));
assertThat(input, hasItem(KV.of(redTeam, 27)));
return null;
});
PAssert.thatMap(teamScores).inOnTimePane(window).isEqualTo(ImmutableMap.<String, Integer>builder().put(redTeam, 7).put(blueTeam, 11).build());
// No final pane is emitted for the blue team, as all of their updates have been taken into
// account in earlier panes
PAssert.that(teamScores).inFinalPane(window).containsInAnyOrder(KV.of(redTeam, 27));
p.run().waitUntilFinish();
}
use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.
the class DoFnSignaturesTest method testWindowParamOnTimer.
@Test
public void testWindowParamOnTimer() throws Exception {
final String timerId = "some-timer-id";
DoFnSignature sig = DoFnSignatures.getSignature(new DoFn<String, String>() {
@TimerId(timerId)
private final TimerSpec myfield1 = TimerSpecs.timer(TimeDomain.EVENT_TIME);
@ProcessElement
public void process(ProcessContext c) {
}
@OnTimer(timerId)
public void onTimer(BoundedWindow w) {
}
}.getClass());
assertThat(sig.onTimerMethods().get(timerId).extraParameters().size(), equalTo(1));
assertThat(sig.onTimerMethods().get(timerId).extraParameters().get(0), instanceOf(WindowParameter.class));
}
use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.
the class DoFnSignaturesSplittableDoFnTest method testSplittableProcessElementMustNotHaveOtherParams.
@Test
public void testSplittableProcessElementMustNotHaveOtherParams() throws Exception {
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Illegal parameter");
thrown.expectMessage("BoundedWindow");
DoFnSignature.ProcessElementMethod signature = analyzeProcessElementMethod(new AnonymousMethod() {
private void method(DoFn<Integer, String>.ProcessContext<Integer, String> context, SomeRestrictionTracker tracker, BoundedWindow window) {
}
});
}
use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.
the class ReduceFnRunner method collectWindows.
/**
* Extract the windows associated with the values.
*/
private Set<W> collectWindows(Iterable<WindowedValue<InputT>> values) throws Exception {
Set<W> windows = new HashSet<>();
for (WindowedValue<?> value : values) {
for (BoundedWindow untypedWindow : value.getWindows()) {
@SuppressWarnings("unchecked") W window = (W) untypedWindow;
windows.add(window);
}
}
return windows;
}
use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project DataflowJavaSDK-examples by GoogleCloudPlatform.
the class LeaderBoardTest method testTeamScoresDroppablyLate.
/**
* A test where elements arrive beyond the maximum allowed lateness. These elements are dropped
* within {@link CalculateTeamScores} and do not impact the final result.
*/
@Test
public void testTeamScoresDroppablyLate() {
BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
TestStream<GameActionInfo> infos = TestStream.create(AvroCoder.of(GameActionInfo.class)).addElements(event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)).advanceWatermarkTo(window.maxTimestamp()).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))).addElements(event(TestUser.BLUE_TWO, 3, TEAM_WINDOW_DURATION.minus(Duration.standardSeconds(5))), event(TestUser.RED_ONE, 7, Duration.standardMinutes(4))).advanceWatermarkToInfinity();
PCollection<KV<String, Integer>> teamScores = p.apply(infos).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
String blueTeam = TestUser.BLUE_ONE.getTeam();
String redTeam = TestUser.RED_ONE.getTeam();
// Only one on-time pane and no late panes should be emitted
PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(redTeam, 7), KV.of(blueTeam, 18));
// No elements are added before the watermark passes the end of the window plus the allowed
// lateness, so no refinement should be emitted
PAssert.that(teamScores).inFinalPane(window).empty();
p.run().waitUntilFinish();
}
Aggregations