Examples with MetricsContainerImpl org.apache.beam.runners.core.metrics.MetricsContainerImpl used on opensource projects

Example 26 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testOnElementBufferingDiscarding.

@Test
public void testOnElementBufferingDiscarding() throws Exception {
    // Test basic execution of a trigger using a non-combining window set and discarding mode.
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Pane of {1, 2}
    injectElement(tester, 1);
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 2);
    assertThat(tester.extractOutput(), contains(isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10)));
    // Pane of just 3, and finish
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    assertThat(tester.extractOutput(), contains(isSingleWindowedValue(containsInAnyOrder(3), 3, 0, 10)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    // This element shouldn't be seen, because the trigger has finished
    injectElement(tester, 4);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(1, droppedElements);
}

Example 27 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class LateDataDroppingDoFnRunnerTest method testLateDataFilter.

@Test
public void testLateDataFilter() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    when(mockTimerInternals.currentInputWatermarkTime()).thenReturn(new Instant(15L));
    LateDataFilter lateDataFilter = new LateDataFilter(WindowingStrategy.of(WINDOW_FN), mockTimerInternals);
    Iterable<WindowedValue<Integer>> actual = lateDataFilter.filter("a", ImmutableList.of(createDatum(13, 13L), // late element, earlier than 4L.
    createDatum(5, 5L), createDatum(16, 16L), createDatum(18, 18L)));
    Iterable<WindowedValue<Integer>> expected = ImmutableList.of(createDatum(13, 13L), createDatum(16, 16L), createDatum(18, 18L));
    assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
    long droppedValues = container.getCounter(MetricName.named(LateDataDroppingDoFnRunner.class, LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS)).getCumulative();
    assertEquals(1, droppedValues);
    // Ensure that reiterating returns the same results and doesn't increment the counter again.
    assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
    droppedValues = container.getCounter(MetricName.named(LateDataDroppingDoFnRunner.class, LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS)).getCumulative();
    assertEquals(1, droppedValues);
}

Example 28 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class DirectTransformExecutor method run.

@Override
public void run() {
    MetricsContainerImpl metricsContainer = new MetricsContainerImpl(transform.getFullName());
    try (Closeable metricsScope = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
        Collection<ModelEnforcement<T>> enforcements = new ArrayList<>();
        for (ModelEnforcementFactory enforcementFactory : modelEnforcements) {
            ModelEnforcement<T> enforcement = enforcementFactory.forBundle(inputBundle, transform);
            enforcements.add(enforcement);
        }
        @Nullable TransformEvaluator<T> evaluator = evaluatorRegistry.forApplication(transform, inputBundle);
        if (evaluator == null) {
            onComplete.handleEmpty(transform);
            // Nothing to do
            return;
        }
        processElements(evaluator, metricsContainer, enforcements);
        finishBundle(evaluator, metricsContainer, enforcements);
    } catch (Exception e) {
        onComplete.handleException(inputBundle, e);
        if (e instanceof RuntimeException) {
            throw (RuntimeException) e;
        }
        throw new RuntimeException(e);
    } catch (Error err) {
        LOG.error("Error occurred within {}", this, err);
        onComplete.handleError(err);
        throw err;
    } finally {
        // Report the physical metrics from the end of this step.
        context.getMetrics().commitPhysical(inputBundle, metricsContainer.getCumulative());
        transformEvaluationState.complete(this);
    }
}

Example 29 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class BatchModeExecutionContext method extractThrottleTime.

public Long extractThrottleTime() {
    long totalThrottleMsecs = 0L;
    for (MetricsContainerImpl container : containerRegistry.getContainers()) {
        // TODO(BEAM-7863): Update throttling counters to use generic throttling-msecs metric.
        CounterCell dataStoreThrottlingTime = container.tryGetCounter(MetricName.named(DATASTORE_THROTTLE_TIME_NAMESPACE, THROTTLE_TIME_COUNTER_NAME));
        if (dataStoreThrottlingTime != null) {
            totalThrottleMsecs += dataStoreThrottlingTime.getCumulative();
        }
        CounterCell httpClientApiThrottlingTime = container.tryGetCounter(MetricName.named(HTTP_CLIENT_API_THROTTLE_TIME_NAMESPACE, THROTTLE_TIME_COUNTER_NAME));
        if (httpClientApiThrottlingTime != null) {
            totalThrottleMsecs += httpClientApiThrottlingTime.getCumulative();
        }
        CounterCell bigqueryStreamingInsertThrottleTime = container.tryGetCounter(MetricName.named(BIGQUERY_STREAMING_INSERT_THROTTLE_TIME_NAMESPACE, THROTTLE_TIME_COUNTER_NAME));
        if (bigqueryStreamingInsertThrottleTime != null) {
            totalThrottleMsecs += bigqueryStreamingInsertThrottleTime.getCumulative();
        }
        CounterCell bigqueryReadThrottleTime = container.tryGetCounter(MetricName.named(BIGQUERY_READ_THROTTLE_TIME_NAMESPACE, THROTTLE_TIME_COUNTER_NAME));
        if (bigqueryReadThrottleTime != null) {
            totalThrottleMsecs += bigqueryReadThrottleTime.getCumulative();
        }
        CounterCell throttlingMsecs = container.tryGetCounter(DataflowSystemMetrics.THROTTLING_MSECS_METRIC_NAME);
        if (throttlingMsecs != null) {
            totalThrottleMsecs += throttlingMsecs.getCumulative();
        }
    }
    return TimeUnit.MILLISECONDS.toSeconds(totalThrottleMsecs);
}

Example 30 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class HarnessMonitoringInfosInstructionHandler method harnessMonitoringInfos.

public BeamFnApi.InstructionResponse.Builder harnessMonitoringInfos(BeamFnApi.InstructionRequest request) {
    BeamFnApi.HarnessMonitoringInfosResponse.Builder response = BeamFnApi.HarnessMonitoringInfosResponse.newBuilder();
    MetricsContainer container = MetricsEnvironment.getProcessWideContainer();
    if (container != null && container instanceof MetricsContainerImpl) {
        response.putAllMonitoringData(((MetricsContainerImpl) container).getMonitoringData(this.metricsShortIds));
    }
    return BeamFnApi.InstructionResponse.newBuilder().setHarnessMonitoringInfos(response);
}