Examples with CounterUpdate com.google.api.services.dataflow.model.CounterUpdate used on opensource projects

Example 26 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class BeamFnMapTaskExecutor method extractMetricUpdates.

/**
 * {@inheritDoc}
 *
 * @return User-defined Beam metrics reported over the Fn API.
 */
@Override
public Iterable<CounterUpdate> extractMetricUpdates() {
    List<CounterUpdate> result = progressTracker.extractCounterUpdates();
    if ((result != null) && (result.size() > 0)) {
        return result;
    }
    // todo(BEAM-6189): Remove this fallback once Metrics is deprecated from SDKs.
    MetricUpdates updates = progressTracker.extractMetricUpdates();
    Iterable<CounterUpdate> deprecatedMetrics = Iterables.concat(StreamSupport.stream(updates.counterUpdates().spliterator(), false).map(update -> MetricsToCounterUpdateConverter.fromCounter(update.getKey(), true, update.getUpdate())).collect(Collectors.toList()), StreamSupport.stream(updates.distributionUpdates().spliterator(), false).map(update -> MetricsToCounterUpdateConverter.fromDistribution(update.getKey(), true, update.getUpdate())).collect(Collectors.toList()));
    return deprecatedMetrics;
}

Example 27 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class UserMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Transforms user counter MonitoringInfo to relevant CounterUpdate.
 *
 * @return Relevant CounterUpdate or null if transformation failed.
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    long value = decodeInt64Counter(monitoringInfo.getPayload());
    Map<String, String> miLabels = monitoringInfo.getLabelsMap();
    final String ptransform = miLabels.get(MonitoringInfoConstants.Labels.PTRANSFORM);
    final String counterName = miLabels.get(MonitoringInfoConstants.Labels.NAME);
    final String counterNamespace = miLabels.get(MonitoringInfoConstants.Labels.NAMESPACE);
    CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
    DataflowStepContext stepContext = transformIdMapping.get(ptransform);
    name.setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setOriginNamespace(counterNamespace)).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(value));
}

Example 28 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class ExecutionTimeMonitoringInfoToCounterUpdateTransformer method transform.

@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    long value = decodeInt64Counter(monitoringInfo.getPayload());
    String urn = monitoringInfo.getUrn();
    final String ptransform = monitoringInfo.getLabelsMap().get(MonitoringInfoConstants.Labels.PTRANSFORM);
    DataflowStepContext stepContext = transformIdMapping.get(ptransform);
    String counterName = URN_TO_COUNTER_NAME_MAPPING.get(urn);
    CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
    name.setName(new CounterStructuredName().setOrigin("SYSTEM").setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setExecutionStepName(stepContext.getNameContext().stageName())).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(value));
}

Example 29 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class MeanByteCountMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Generates CounterUpdate to send to DFE based on ElementCount MonitoringInfo.
 *
 * @param monitoringInfo Monitoring info to transform.
 * @return CounterUpdate generated based on provided monitoringInfo
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
    final String pcollectionId = monitoringInfo.getLabelsMap().get(MonitoringInfoConstants.Labels.PCOLLECTION);
    final String pcollectionName = pcollectionIdToNameContext.get(pcollectionId).userName();
    String counterName = pcollectionName + "-MeanByteCount";
    NameAndKind name = new NameAndKind();
    name.setName(counterName).setKind(Kind.MEAN.toString());
    return new CounterUpdate().setNameAndKind(name).setCumulative(true).setIntegerMean(new IntegerMean().setSum(longToSplitInt(data.sum())).setCount(longToSplitInt(data.count())));
}

Example 30 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class UserDistributionMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Transforms user counter MonitoringInfo to relevant CounterUpdate.
 *
 * @return Relevant CounterUpdate or null if transformation failed.
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
    Map<String, String> miLabels = monitoringInfo.getLabelsMap();
    final String ptransform = miLabels.get(MonitoringInfoConstants.Labels.PTRANSFORM);
    final String counterName = miLabels.get(MonitoringInfoConstants.Labels.NAME);
    final String counterNamespace = miLabels.get(MonitoringInfoConstants.Labels.NAMESPACE);
    CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
    DataflowStepContext stepContext = transformIdMapping.get(ptransform);
    name.setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setOriginNamespace(counterNamespace)).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setDistribution(new DistributionUpdate().setMax(DataflowCounterUpdateExtractor.longToSplitInt(data.max())).setMin(DataflowCounterUpdateExtractor.longToSplitInt(data.min())).setSum(DataflowCounterUpdateExtractor.longToSplitInt(data.sum())).setCount(DataflowCounterUpdateExtractor.longToSplitInt(data.count())));
}