Search in sources :

Example 16 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class SpannerIOWriteTest method verifyMetricWasSet.

private void verifyMetricWasSet(String projectId, String databaseId, String tableId, String method, String status, long count) {
    // Verify the metric was reported.
    HashMap<String, String> labels = new HashMap<>();
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "Spanner");
    labels.put(MonitoringInfoConstants.Labels.METHOD, method);
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.spannerTable(projectId, databaseId, tableId));
    labels.put(MonitoringInfoConstants.Labels.SPANNER_PROJECT_ID, projectId);
    labels.put(MonitoringInfoConstants.Labels.SPANNER_DATABASE_ID, databaseId);
    labels.put(MonitoringInfoConstants.Labels.SPANNER_INSTANCE_ID, tableId);
    labels.put(MonitoringInfoConstants.Labels.STATUS, status);
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
    assertEquals(count, (long) container.getCounter(name).getCumulative());
}
Also used : MonitoringInfoMetricName(org.apache.beam.runners.core.metrics.MonitoringInfoMetricName) MetricsContainerImpl(org.apache.beam.runners.core.metrics.MetricsContainerImpl) HashMap(java.util.HashMap)

Example 17 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class DatastoreV1Test method verifyMetricWasSet.

private void verifyMetricWasSet(String method, String status, String namespace, long count) {
    // Verify the metric as reported.
    HashMap<String, String> labels = new HashMap<>();
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "Datastore");
    labels.put(MonitoringInfoConstants.Labels.METHOD, method);
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, "//bigtable.googleapis.com/projects/" + PROJECT_ID + "/namespaces/" + namespace);
    labels.put(MonitoringInfoConstants.Labels.DATASTORE_PROJECT, PROJECT_ID);
    labels.put(MonitoringInfoConstants.Labels.DATASTORE_NAMESPACE, namespace);
    labels.put(MonitoringInfoConstants.Labels.STATUS, status);
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
    assertEquals(count, (long) container.getCounter(name).getCumulative());
}
Also used : MonitoringInfoMetricName(org.apache.beam.runners.core.metrics.MonitoringInfoMetricName) MetricsContainerImpl(org.apache.beam.runners.core.metrics.MetricsContainerImpl) HashMap(java.util.HashMap)

Example 18 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class TransformExecutor method run.

@Override
public void run() {
    MetricsContainerImpl metricsContainer = new MetricsContainerImpl(transform.getFullName());
    try (Closeable metricsScope = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
        Collection<ModelEnforcement<T>> enforcements = new ArrayList<>();
        for (ModelEnforcementFactory enforcementFactory : modelEnforcements) {
            ModelEnforcement<T> enforcement = enforcementFactory.forBundle(inputBundle, transform);
            enforcements.add(enforcement);
        }
        TransformEvaluator<T> evaluator = evaluatorFactory.forApplication(transform, inputBundle);
        if (evaluator == null) {
            onComplete.handleEmpty(transform);
            // Nothing to do
            return;
        }
        processElements(evaluator, metricsContainer, enforcements);
        finishBundle(evaluator, metricsContainer, enforcements);
    } catch (Exception e) {
        onComplete.handleException(inputBundle, e);
        if (e instanceof RuntimeException) {
            throw (RuntimeException) e;
        }
        throw new RuntimeException(e);
    } catch (Error err) {
        LOG.error("Error occurred within {}", this, err);
        onComplete.handleError(err);
        throw err;
    } finally {
        // Report the physical metrics from the end of this step.
        context.getMetrics().commitPhysical(inputBundle, metricsContainer.getCumulative());
        transformEvaluationState.complete(this);
    }
}
Also used : MetricsContainerImpl(org.apache.beam.runners.core.metrics.MetricsContainerImpl) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList)

Example 19 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class FlinkMetricContainerTest method testDropUnexpectedMonitoringInfoTypes.

@Test
public void testDropUnexpectedMonitoringInfoTypes() {
    MetricsContainerImpl step = container.getMetricsContainer("step");
    MonitoringInfo intCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns1").setLabel(MonitoringInfoConstants.Labels.NAME, "int_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64SumValue(111).build();
    MonitoringInfo doubleCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns2").setLabel(MonitoringInfoConstants.Labels.NAME, "double_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleSumValue(222).build();
    MonitoringInfo intDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns3").setLabel(MonitoringInfoConstants.Labels.NAME, "int_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64DistributionValue(DistributionData.create(30, 10, 1, 5)).build();
    MonitoringInfo doubleDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns4").setLabel(MonitoringInfoConstants.Labels.NAME, "double_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleDistributionValue(10, 30, 1, 5).build();
    // Mock out the counter that Flink returns; the distribution gets created by
    // FlinkMetricContainer, not by Flink itself, so we verify it in a different way below
    SimpleCounter counter = new SimpleCounter();
    when(metricGroup.counter("ns1.int_counter")).thenReturn(counter);
    container.updateMetrics("step", ImmutableList.of(intCounter, doubleCounter, intDistribution, doubleDistribution));
    // Flink's MetricGroup should only have asked for one counter (the integer-typed one) to be
    // created (the double-typed one is dropped currently)
    verify(metricGroup).counter(eq("ns1.int_counter"));
    // Verify that the counter injected into flink has the right value
    assertThat(counter.getCount(), is(111L));
    // Verify the counter in the java SDK MetricsContainer
    long count = ((CounterCell) step.tryGetCounter(MonitoringInfoMetricName.of(intCounter))).getCumulative();
    assertThat(count, is(111L));
    // The one Flink distribution that gets created is a FlinkDistributionGauge; here we verify its
    // initial (and in this test, final) value
    verify(metricGroup).gauge(eq("ns3.int_distribution"), argThat(new ArgumentMatcher<FlinkDistributionGauge>() {

        @Override
        public boolean matches(FlinkDistributionGauge argument) {
            DistributionResult actual = ((FlinkDistributionGauge) argument).getValue();
            DistributionResult expected = DistributionResult.create(30, 10, 1, 5);
            return actual.equals(expected);
        }
    }));
    // Verify that the Java SDK MetricsContainer holds the same information
    DistributionData distributionData = ((DistributionCell) step.getDistribution(MonitoringInfoMetricName.of(intDistribution))).getCumulative();
    assertThat(distributionData, is(DistributionData.create(30, 10, 1, 5)));
}
Also used : MetricsContainerImpl(org.apache.beam.runners.core.metrics.MetricsContainerImpl) CounterCell(org.apache.beam.runners.core.metrics.CounterCell) DistributionResult(org.apache.beam.sdk.metrics.DistributionResult) MonitoringInfo(org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo) SimpleMonitoringInfoBuilder(org.apache.beam.runners.core.metrics.SimpleMonitoringInfoBuilder) SimpleCounter(org.apache.flink.metrics.SimpleCounter) DistributionData(org.apache.beam.runners.core.metrics.DistributionData) ArgumentMatcher(org.mockito.ArgumentMatcher) FlinkDistributionGauge(org.apache.beam.runners.flink.metrics.FlinkMetricContainer.FlinkDistributionGauge) DistributionCell(org.apache.beam.runners.core.metrics.DistributionCell) Test(org.junit.Test)

Example 20 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class SimplePushbackSideInputDoFnRunnerTest method testLateDroppingForStatefulDoFnRunner.

@Test
@Category({ ValidatesRunner.class })
public void testLateDroppingForStatefulDoFnRunner() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    timerInternals.advanceOutputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    PushbackSideInputDoFnRunner runner = createRunner(statefulRunner, ImmutableList.of(singletonView));
    runner.startBundle();
    when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class))).thenReturn(true);
    WindowedValue<Integer> multiWindow = WindowedValue.of(1, new Instant(0), ImmutableList.of(new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE))), PaneInfo.ON_TIME_AND_ONLY_FIRING);
    runner.processElementInReadyWindows(multiWindow);
    long droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative();
    assertEquals(1L, droppedValues);
    runner.finishBundle();
}
Also used : MetricsContainerImpl(org.apache.beam.runners.core.metrics.MetricsContainerImpl) Instant(org.joda.time.Instant) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Aggregations

MetricsContainerImpl (org.apache.beam.runners.core.metrics.MetricsContainerImpl)36 Test (org.junit.Test)16 IntervalWindow (org.apache.beam.sdk.transforms.windowing.IntervalWindow)12 Instant (org.joda.time.Instant)12 HashMap (java.util.HashMap)7 MonitoringInfoMetricName (org.apache.beam.runners.core.metrics.MonitoringInfoMetricName)7 Matchers.emptyIterable (org.hamcrest.Matchers.emptyIterable)7 WindowedValue (org.apache.beam.sdk.util.WindowedValue)6 KV (org.apache.beam.sdk.values.KV)5 Before (org.junit.Before)5 Closeable (java.io.Closeable)4 ArrayList (java.util.ArrayList)4 CounterCell (org.apache.beam.runners.core.metrics.CounterCell)4 WindowMatchers.isSingleWindowedValue (org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue)3 WindowMatchers.isWindowedValue (org.apache.beam.runners.core.WindowMatchers.isWindowedValue)3 Duration (org.joda.time.Duration)3 ByteString (com.google.protobuf.ByteString)2 IOException (java.io.IOException)2 CounterSet (org.apache.beam.runners.dataflow.worker.counters.CounterSet)2 TupleTag (org.apache.beam.sdk.values.TupleTag)2