Examples with MonitoringInfo org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo used on opensource projects

Example 16 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class MeanByteCountMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsNullIfMonitoringInfoWithUnknownPCollectionLabelPresent.

@Test
public void testTransformReturnsNullIfMonitoringInfoWithUnknownPCollectionLabelPresent() {
    Map<String, NameContext> pcollectionNameMapping = new HashMap<>();
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.SAMPLED_BYTE_SIZE).setType(TypeUrns.DISTRIBUTION_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PCOLLECTION, "anyValue").build();
    MeanByteCountMonitoringInfoToCounterUpdateTransformer testObject = new MeanByteCountMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, pcollectionNameMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    assertNull(testObject.transform(monitoringInfo));
}

Example 17 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class MetricsContainerImpl method getShortId.

private String getShortId(MetricKey key, Function<MetricKey, SimpleMonitoringInfoBuilder> toInfo, ShortIdMap shortIds) {
    Optional<String> shortId = shortIdsByMetricKey.get(key);
    if (shortId == null) {
        SimpleMonitoringInfoBuilder monitoringInfoBuilder = toInfo.apply(key);
        if (monitoringInfoBuilder == null) {
            shortId = Optional.empty();
        } else {
            MonitoringInfo monitoringInfo = monitoringInfoBuilder.build();
            if (monitoringInfo == null) {
                shortId = Optional.empty();
            } else {
                shortId = Optional.of(shortIds.getOrCreateShortId(monitoringInfo));
            }
        }
        shortIdsByMetricKey.put(key, shortId);
    }
    return shortId.orElse(null);
}

Example 18 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class MetricsContainerImpl method getMonitoringInfos.

/**
 * Return the cumulative values for any metrics in this container as MonitoringInfos.
 */
@Override
public Iterable<MonitoringInfo> getMonitoringInfos() {
    // Extract user metrics and store as MonitoringInfos.
    ArrayList<MonitoringInfo> monitoringInfos = new ArrayList<MonitoringInfo>();
    MetricUpdates metricUpdates = this.getUpdates();
    for (MetricUpdate<Long> metricUpdate : metricUpdates.counterUpdates()) {
        MonitoringInfo mi = counterUpdateToMonitoringInfo(metricUpdate);
        if (mi != null) {
            monitoringInfos.add(mi);
        }
    }
    for (MetricUpdate<org.apache.beam.runners.core.metrics.DistributionData> metricUpdate : metricUpdates.distributionUpdates()) {
        MonitoringInfo mi = distributionUpdateToMonitoringInfo(metricUpdate);
        if (mi != null) {
            monitoringInfos.add(mi);
        }
    }
    return monitoringInfos;
}

Example 19 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class PCollectionConsumerRegistryTest method multipleConsumersSamePCollection.

/**
 * Test that the counter increments only once when multiple consumers of same pCollection read the
 * same element.
 */
@Test
public void multipleConsumersSamePCollection() throws Exception {
    final String pCollectionA = "pCollectionA";
    final String pTransformIdA = "pTransformIdA";
    final String pTransformIdB = "pTransformIdB";
    MetricsContainerStepMap metricsContainerRegistry = new MetricsContainerStepMap();
    PCollectionConsumerRegistry consumers = new PCollectionConsumerRegistry(metricsContainerRegistry, mock(ExecutionStateTracker.class));
    FnDataReceiver<WindowedValue<String>> consumerA1 = mock(FnDataReceiver.class);
    FnDataReceiver<WindowedValue<String>> consumerA2 = mock(FnDataReceiver.class);
    consumers.register(pCollectionA, pTransformIdA, consumerA1, StringUtf8Coder.of());
    consumers.register(pCollectionA, pTransformIdB, consumerA2, StringUtf8Coder.of());
    FnDataReceiver<WindowedValue<String>> wrapperConsumer = (FnDataReceiver<WindowedValue<String>>) (FnDataReceiver) consumers.getMultiplexingConsumer(pCollectionA);
    WindowedValue<String> element = valueInGlobalWindow("elem");
    int numElements = 20;
    for (int i = 0; i < numElements; i++) {
        wrapperConsumer.accept(element);
    }
    // Check that the underlying consumers are each invoked per element.
    verify(consumerA1, times(numElements)).accept(element);
    verify(consumerA2, times(numElements)).accept(element);
    assertThat(consumers.keySet(), contains(pCollectionA));
    SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT);
    builder.setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, pCollectionA);
    builder.setInt64SumValue(numElements);
    MonitoringInfo expected = builder.build();
    // Clear the timestamp before comparison.
    MonitoringInfo result = Iterables.find(metricsContainerRegistry.getMonitoringInfos(), monitoringInfo -> monitoringInfo.containsLabels(Labels.PCOLLECTION));
    assertEquals(expected, result);
}

Example 20 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class ProcessBundleHandler method monitoringData.

private ImmutableMap<String, ByteString> monitoringData(BundleProcessor bundleProcessor) throws Exception {
    ImmutableMap.Builder<String, ByteString> result = ImmutableMap.builder();
    // Get start bundle Execution Time Metrics.
    result.putAll(bundleProcessor.getStartFunctionRegistry().getExecutionTimeMonitoringData(shortIds));
    // Get process bundle Execution Time Metrics.
    result.putAll(bundleProcessor.getpCollectionConsumerRegistry().getExecutionTimeMonitoringData(shortIds));
    // Get finish bundle Execution Time Metrics.
    result.putAll(bundleProcessor.getFinishFunctionRegistry().getExecutionTimeMonitoringData(shortIds));
    // Extract MonitoringInfos that come from the metrics container registry.
    result.putAll(bundleProcessor.getMetricsContainerRegistry().getMonitoringData(shortIds));
    // Add any additional monitoring infos that the "runners" report explicitly.
    for (ProgressRequestCallback progressRequestCallback : bundleProcessor.getProgressRequestCallbacks()) {
        // TODO(BEAM-6597): Plumb reporting monitoring infos using the short id system upstream.
        for (MetricsApi.MonitoringInfo monitoringInfo : progressRequestCallback.getMonitoringInfos()) {
            ByteString payload = monitoringInfo.getPayload();
            String shortId = shortIds.getOrCreateShortId(monitoringInfo.toBuilder().clearPayload().build());
            result.put(shortId, payload);
        }
    }
    return result.build();
}