Examples with MetricUpdates org.apache.beam.runners.core.metrics.MetricUpdates used on opensource projects

Example 1 with MetricUpdates

use of org.apache.beam.runners.core.metrics.MetricUpdates in project beam by apache.

the class BeamFnMapTaskExecutor method extractMetricUpdates.

/**
 * {@inheritDoc}
 *
 * @return User-defined Beam metrics reported over the Fn API.
 */
@Override
public Iterable<CounterUpdate> extractMetricUpdates() {
    List<CounterUpdate> result = progressTracker.extractCounterUpdates();
    if ((result != null) && (result.size() > 0)) {
        return result;
    }
    // todo(BEAM-6189): Remove this fallback once Metrics is deprecated from SDKs.
    MetricUpdates updates = progressTracker.extractMetricUpdates();
    Iterable<CounterUpdate> deprecatedMetrics = Iterables.concat(StreamSupport.stream(updates.counterUpdates().spliterator(), false).map(update -> MetricsToCounterUpdateConverter.fromCounter(update.getKey(), true, update.getUpdate())).collect(Collectors.toList()), StreamSupport.stream(updates.distributionUpdates().spliterator(), false).map(update -> MetricsToCounterUpdateConverter.fromDistribution(update.getKey(), true, update.getUpdate())).collect(Collectors.toList()));
    return deprecatedMetrics;
}

Example 2 with MetricUpdates

use of org.apache.beam.runners.core.metrics.MetricUpdates in project beam by apache.

the class JetMetricsContainer method flush.

@SuppressWarnings("FutureReturnValueIgnored")
public void flush(boolean async) {
    if (counters.isEmpty() && distributions.isEmpty() && gauges.isEmpty()) {
        return;
    }
    ImmutableList<MetricUpdates.MetricUpdate<Long>> counters = extractUpdates(this.counters);
    ImmutableList<MetricUpdates.MetricUpdate<DistributionData>> distributions = extractUpdates(this.distributions);
    ImmutableList<MetricUpdates.MetricUpdate<GaugeData>> gauges = extractUpdates(this.gauges);
    MetricUpdates updates = new MetricUpdatesImpl(counters, distributions, gauges);
    if (async) {
        accumulator.setAsync(metricsKey, updates);
    } else {
        accumulator.set(metricsKey, updates);
    }
}

Example 3 with MetricUpdates

use of org.apache.beam.runners.core.metrics.MetricUpdates in project beam by apache.

the class TransformExecutor method processElements.

/**
   * Processes all the elements in the input bundle using the transform evaluator, applying any
   * necessary {@link ModelEnforcement ModelEnforcements}.
   */
private void processElements(TransformEvaluator<T> evaluator, MetricsContainerImpl metricsContainer, Collection<ModelEnforcement<T>> enforcements) throws Exception {
    if (inputBundle != null) {
        for (WindowedValue<T> value : inputBundle.getElements()) {
            for (ModelEnforcement<T> enforcement : enforcements) {
                enforcement.beforeElement(value);
            }
            evaluator.processElement(value);
            // Report the physical metrics after each element
            MetricUpdates deltas = metricsContainer.getUpdates();
            if (deltas != null) {
                context.getMetrics().updatePhysical(inputBundle, deltas);
                metricsContainer.commitUpdates();
            }
            for (ModelEnforcement<T> enforcement : enforcements) {
                enforcement.afterElement(value);
            }
        }
    }
}

Example 4 with MetricUpdates

use of org.apache.beam.runners.core.metrics.MetricUpdates in project beam by apache.

the class DirectTransformExecutor method processElements.

/**
 * Processes all the elements in the input bundle using the transform evaluator, applying any
 * necessary {@link ModelEnforcement ModelEnforcements}.
 */
private void processElements(TransformEvaluator<T> evaluator, MetricsContainerImpl metricsContainer, Collection<ModelEnforcement<T>> enforcements) throws Exception {
    if (inputBundle != null) {
        for (WindowedValue<T> value : inputBundle.getElements()) {
            for (ModelEnforcement<T> enforcement : enforcements) {
                enforcement.beforeElement(value);
            }
            evaluator.processElement(value);
            // Report the physical metrics after each element
            MetricUpdates deltas = metricsContainer.getUpdates();
            if (deltas != null) {
                context.getMetrics().updatePhysical(inputBundle, deltas);
                metricsContainer.commitUpdates();
            }
            for (ModelEnforcement<T> enforcement : enforcements) {
                enforcement.afterElement(value);
            }
        }
    }
}

Example 5 with MetricUpdates

use of org.apache.beam.runners.core.metrics.MetricUpdates in project beam by apache.

the class JetRunner method run.

private JetPipelineResult run(DAG dag) {
    startClusterIfNeeded(options);
    JetInstance jet = getJetInstance(// todo: we use single client for each job, it might be better to have a
    options);
    // shared client with refcount
    Job job = jet.newJob(dag, getJobConfig(options));
    IMap<String, MetricUpdates> metricsAccumulator = jet.getMap(JetMetricsContainer.getMetricsMapName(job.getId()));
    JetPipelineResult pipelineResult = new JetPipelineResult(job, metricsAccumulator);
    CompletableFuture<Void> completionFuture = job.getFuture().whenCompleteAsync((r, f) -> {
        pipelineResult.freeze(f);
        metricsAccumulator.destroy();
        jet.shutdown();
        stopClusterIfNeeded(options);
    });
    pipelineResult.setCompletionFuture(completionFuture);
    return pipelineResult;
}