Examples with MetricsContainerImpl org.apache.beam.runners.core.metrics.MetricsContainerImpl used on opensource projects

Example 11 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testIdempotentEmptyPanesDiscarding.

@Test
public void testIdempotentEmptyPanesDiscarding() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test uninteresting (empty) panes don't increment the index or otherwise
    // modify PaneInfo.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Inject a couple of on-time elements and fire at the window end.
    injectElement(tester, 1);
    injectElement(tester, 2);
    tester.advanceInputWatermark(new Instant(12));
    // Fire the on-time pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Fire another timer (with no data, so it's an uninteresting pane that should not be output).
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Finish it off with another datum.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    // The intermediate trigger firing shouldn't result in any output.
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(2));
    // The on-time pane is as expected.
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10));
    // The late pane has the correct indices.
    assertThat(output.get(1).getValue(), contains(3));
    assertThat(output.get(1).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 1, 1)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
}

Example 12 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testWatermarkHoldAndLateData.

@Test
public void testWatermarkHoldAndLateData() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test handling of late data. Specifically, ensure the watermark hold is correct.
    Duration allowedLateness = Duration.millis(10);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, allowedLateness, ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Input watermark -> null
    assertEquals(null, tester.getWatermarkHold());
    assertEquals(null, tester.getOutputWatermark());
    // All on time data, verify watermark hold.
    IntervalWindow expectedWindow = new IntervalWindow(new Instant(0), new Instant(10));
    injectElement(tester, 1);
    injectElement(tester, 3);
    assertEquals(new Instant(1), tester.getWatermarkHold());
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 2);
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(1, 2, 3), equalTo(new Instant(1)), equalTo((BoundedWindow) expectedWindow))));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    // There is no end-of-window hold, but the timer set by the trigger holds the watermark
    assertThat(tester.getWatermarkHold(), nullValue());
    // Nothing dropped.
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
    // Input watermark -> 4, output watermark should advance that far as well
    tester.advanceInputWatermark(new Instant(4));
    assertEquals(new Instant(4), tester.getOutputWatermark());
    // Some late, some on time. Verify that we only hold to the minimum of on-time.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(4));
    injectElement(tester, 2);
    injectElement(tester, 3);
    // Late data has arrived behind the _output_ watermark. The ReduceFnRunner sets a GC hold
    // since this data is not permitted to hold up the output watermark.
    assertThat(tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp().plus(allowedLateness)));
    // Now data just ahead of the output watermark arrives and sets an earlier "element" hold
    injectElement(tester, 5);
    assertEquals(new Instant(5), tester.getWatermarkHold());
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 4);
    output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, 2, 3, 4, // new elements
    5), // timestamp
    4, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, false, Timing.EARLY, 1, -1)));
    // Since the element hold is cleared, there is no hold remaining
    assertThat(tester.getWatermarkHold(), nullValue());
    // All behind the output watermark -- hold is at GC time (if we imagine the
    // trigger sets a timer for ON_TIME firing, that is actually when they'll be emitted)
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(8));
    injectElement(tester, 6);
    injectElement(tester, 5);
    assertThat(tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp().plus(allowedLateness)));
    injectElement(tester, 4);
    // Fire the ON_TIME pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    // To get an ON_TIME pane, we need the output watermark to be held back a little; this would
    // be done by way of the timers set by the trigger, which are mocked here
    tester.setAutoAdvanceOutputWatermark(false);
    tester.advanceInputWatermark(expectedWindow.maxTimestamp().plus(Duration.millis(1)));
    tester.fireTimer(expectedWindow, expectedWindow.maxTimestamp(), TimeDomain.EVENT_TIME);
    // Output time is end of the window, because all the new data was late, but the pane
    // is the ON_TIME pane.
    output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, // earlier firing
    2, // earlier firing
    3, // earlier firing
    4, // earlier firing
    5, 4, 5, // new elements
    6), // timestamp
    9, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, false, Timing.ON_TIME, 2, 0)));
    tester.setAutoAdvanceOutputWatermark(true);
    // This is "pending" at the time the watermark makes it way-late.
    // Because we're about to expire the window, we output it.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    injectElement(tester, 8);
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
    // Exceed the GC limit, triggering the last pane to be fired
    tester.advanceInputWatermark(new Instant(50));
    output = tester.extractOutput();
    // Output time is still end of the window, because the new data (8) was behind
    // the output watermark.
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, // earlier firing
    2, // earlier firing
    3, // earlier firing
    4, // earlier firing
    5, // earlier firing
    4, // earlier firing
    5, // earlier firing
    6, // new element prior to window becoming expired
    8), // timestamp
    9, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 3, 1)));
    assertEquals(new Instant(50), tester.getOutputWatermark());
    assertEquals(null, tester.getWatermarkHold());
    // Late timers are ignored
    tester.fireTimer(new IntervalWindow(new Instant(0), new Instant(10)), new Instant(12), TimeDomain.EVENT_TIME);
    // And because we're past the end of window + allowed lateness, everything should be cleaned up.
    assertFalse(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor();
}

Example 13 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testDropDataMultipleWindowsFinishedTrigger.

/**
 * Tests that when data is assigned to multiple windows but some of those windows have had their
 * triggers finish, then the data is dropped and counted accurately.
 */
@Test
public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(WindowingStrategy.of(SlidingWindows.of(Duration.millis(100)).every(Duration.millis(30))).withTrigger(AfterWatermark.pastEndOfWindow()).withAllowedLateness(Duration.millis(1000)), Sum.ofIntegers(), VarIntCoder.of());
    tester.injectElements(// assigned to [-60, 40), [-30, 70), [0, 100)
    TimestampedValue.of(10, new Instant(23)), // assigned to [-30, 70), [0, 100), [30, 130)
    TimestampedValue.of(12, new Instant(40)));
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
    tester.advanceInputWatermark(new Instant(70));
    tester.injectElements(// but [-30, 70) is closed by the trigger
    TimestampedValue.of(14, new Instant(60)));
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(1, droppedElements);
    tester.advanceInputWatermark(new Instant(130));
    // assigned to [-30, 70), [0, 100), [30, 130)
    // but they are all closed
    tester.injectElements(TimestampedValue.of(16, new Instant(40)));
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(4, droppedElements);
}

Example 14 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class StatefulDoFnRunnerTest method testLateDropping.

private void testLateDropping(boolean ordered) throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    timerInternals.advanceOutputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    MyDoFn fn = MyDoFn.create(ordered);
    DoFnRunner<KV<String, Integer>, Integer> runner = createStatefulDoFnRunner(fn);
    runner.startBundle();
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE));
    Instant timestamp = new Instant(0);
    runner.processElement(WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING));
    long droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative();
    assertEquals(1L, droppedValues);
    runner.finishBundle();
}

Example 15 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class SpannerIOReadTest method verifyMetricWasSet.

private void verifyMetricWasSet(String projectId, String databaseId, String tableId, String status, @Nullable String queryName, long count) {
    // Verify the metric was reported.
    HashMap<String, String> labels = new HashMap<>();
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "Spanner");
    labels.put(MonitoringInfoConstants.Labels.METHOD, "Read");
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.spannerTable(projectId, databaseId, tableId));
    labels.put(MonitoringInfoConstants.Labels.SPANNER_PROJECT_ID, projectId);
    labels.put(MonitoringInfoConstants.Labels.SPANNER_DATABASE_ID, databaseId);
    labels.put(MonitoringInfoConstants.Labels.SPANNER_INSTANCE_ID, tableId);
    if (queryName != null) {
        labels.put(MonitoringInfoConstants.Labels.SPANNER_QUERY_NAME, queryName);
    }
    labels.put(MonitoringInfoConstants.Labels.STATUS, status);
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
    assertEquals(count, (long) container.getCounter(name).getCumulative());
}