Examples with MetricsContainerImpl org.apache.beam.runners.core.metrics.MetricsContainerImpl used on opensource projects

Example 21 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class StatefulDoFnRunnerTest method testDataDroppedBasedOnInputWatermarkWhenOrdered.

@Test
public void testDataDroppedBasedOnInputWatermarkWhenOrdered() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    Instant timestamp = new Instant(0);
    MyDoFn fn = MyDoFn.create(true);
    DoFnRunner<KV<String, Integer>, Integer> runner = createStatefulDoFnRunner(fn);
    runner.startBundle();
    IntervalWindow window = new IntervalWindow(timestamp, timestamp.plus(Duration.millis(WINDOW_SIZE)));
    runner.processElement(WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING));
    long droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative();
    assertEquals(0L, droppedValues);
    timerInternals.advanceInputWatermark(timestamp.plus(Duration.millis(ALLOWED_LATENESS + 1)));
    runner.processElement(WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING));
    droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative();
    assertEquals(1L, droppedValues);
    runner.finishBundle();
}

Example 22 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class SparkExecutableStageFunction method getBundleProgressHandler.

private BundleProgressHandler getBundleProgressHandler() {
    String stageName = stagePayload.getInput();
    MetricsContainerImpl container = metricsAccumulator.value().getContainer(stageName);
    return new BundleProgressHandler() {

        @Override
        public void onProgress(ProcessBundleProgressResponse progress) {
            container.update(progress.getMonitoringInfosList());
        }

        @Override
        public void onCompleted(ProcessBundleResponse response) {
            container.update(response.getMonitoringInfosList());
        }
    };
}

Example 23 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class GcsUtilTest method setUp.

@Before
public void setUp() {
    // Setup the ProcessWideContainer for testing metrics are set.
    MetricsContainerImpl container = new MetricsContainerImpl(null);
    MetricsEnvironment.setProcessWideContainer(container);
}

Example 24 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testIdempotentEmptyPanesAccumulating.

@Test
public void testIdempotentEmptyPanesAccumulating() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test uninteresting (empty) panes don't increment the index or otherwise
    // modify PaneInfo.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Inject a couple of on-time elements and fire at the window end.
    injectElement(tester, 1);
    injectElement(tester, 2);
    tester.advanceInputWatermark(new Instant(12));
    // Trigger the on-time pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0)));
    // Fire another timer with no data; the empty pane should not be output even though the
    // trigger is ready to fire
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    assertThat(tester.extractOutput().size(), equalTo(0));
    // Finish it off with another datum, which is late
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    // The late pane has the correct indices.
    assertThat(output.get(0).getValue(), containsInAnyOrder(1, 2, 3));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 1, 1)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
}

Example 25 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testMergingWatermarkHoldAndLateDataFuzz.

@Test
public void testMergingWatermarkHoldAndLateDataFuzz() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test handling of late data. Specifically, ensure the watermark hold is correct.
    Duration allowedLateness = Duration.standardMinutes(100);
    long seed = ThreadLocalRandom.current().nextLong();
    LOG.info("Random seed: {}", seed);
    Random r = new Random(seed);
    Duration gapDuration = Duration.millis(10 + r.nextInt(40));
    LOG.info("Gap duration {}", gapDuration);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(Sessions.withGapDuration(gapDuration)).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(1)))).withAllowedLateness(allowedLateness));
    tester.setAutoAdvanceOutputWatermark(true);
    // Input watermark -> null
    assertEquals(null, tester.getWatermarkHold());
    assertEquals(null, tester.getOutputWatermark());
    // All on time data, verify watermark hold.
    List<Integer> times = new ArrayList<>();
    int numTs = 3 + r.nextInt(100);
    int maxTs = 1 + r.nextInt(400);
    LOG.info("Num ts {}", numTs);
    LOG.info("Max ts {}", maxTs);
    for (int i = numTs; i >= 0; --i) {
        times.add(r.nextInt(maxTs));
    }
    LOG.info("Times: {}", times);
    int split = 0;
    long watermark = 0;
    while (split < times.size()) {
        int nextSplit = split + r.nextInt(times.size());
        if (nextSplit > times.size()) {
            nextSplit = times.size();
        }
        LOG.info("nextSplit {}", nextSplit);
        injectElements(tester, times.subList(split, nextSplit));
        if (r.nextInt(3) == 0) {
            int nextWatermark = r.nextInt((int) (maxTs + gapDuration.getMillis()));
            if (nextWatermark > watermark) {
                Boolean enabled = r.nextBoolean();
                LOG.info("nextWatermark {} {}", nextWatermark, enabled);
                watermark = nextWatermark;
                tester.setAutoAdvanceOutputWatermark(enabled);
                tester.advanceInputWatermark(new Instant(watermark));
            }
        }
        split = nextSplit;
        Instant hold = tester.getWatermarkHold();
        if (hold != null) {
            assertThat(hold, greaterThanOrEqualTo(new Instant(watermark)));
            assertThat(watermark, lessThan(maxTs + gapDuration.getMillis()));
        }
    }
    tester.setAutoAdvanceOutputWatermark(true);
    watermark = gapDuration.getMillis() + maxTs;
    tester.advanceInputWatermark(new Instant(watermark));
    LOG.info("Output {}", tester.extractOutput());
    if (tester.getWatermarkHold() != null) {
        assertThat(tester.getWatermarkHold(), equalTo(new Instant(watermark).plus(allowedLateness)));
    }
    // Nothing dropped.
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative().longValue();
    assertEquals(0, droppedElements);
}