Examples with CounterMetadata com.google.api.services.dataflow.model.CounterMetadata used on opensource projects

Example 6 with CounterMetadata

use of com.google.api.services.dataflow.model.CounterMetadata in project beam by apache.

the class CounterUpdateAggregatorsTest method testAggregateMean.

@Test
public void testAggregateMean() {
    List<CounterUpdate> meanUpdates = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        meanUpdates.add(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.MEAN.toString()))).setIntegerMean(new IntegerMean().setSum(longToSplitInt((long) i)).setCount(longToSplitInt(1L))));
    }
    List<CounterUpdate> aggregated = CounterUpdateAggregators.aggregate(meanUpdates);
    assertEquals(1, aggregated.size());
    CounterUpdate combined = aggregated.get(0);
    assertEquals(45L, splitIntToLong(combined.getIntegerMean().getSum()));
    assertEquals(10L, splitIntToLong(combined.getIntegerMean().getCount()));
}

Example 7 with CounterMetadata

use of com.google.api.services.dataflow.model.CounterMetadata in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMsecCounter.

@Test
public void populateCounterUpdatesWithMsecCounter() throws Exception {
    final CounterUpdate expectedMsec = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("SYSTEM").setName("start-msecs").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of());
    when(context.extractMsecCounters(anyBoolean())).thenReturn(ImmutableList.of(expectedMsec));
    WorkItemStatus status = new WorkItemStatus();
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedMsec));
}

Example 8 with CounterMetadata

use of com.google.api.services.dataflow.model.CounterMetadata in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMetricsAndCounters.

/**
 * Validates that Beam Metrics and "internal" Counters are merged in the update.
 */
@Test
public void populateCounterUpdatesWithMetricsAndCounters() throws Exception {
    final CounterUpdate expectedCounter = new CounterUpdate().setNameAndKind(new NameAndKind().setName("some-counter").setKind(Kind.SUM.toString())).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    CounterSet counterSet = new CounterSet();
    counterSet.intSum(CounterName.named("some-counter")).addValue(42);
    final CounterUpdate expectedMetric = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-counter").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    MetricsContainerImpl metricsContainer = new MetricsContainerImpl("step");
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of(expectedMetric));
    when(context.extractMsecCounters(anyBoolean())).thenReturn(Collections.emptyList());
    CounterCell counter = metricsContainer.getCounter(MetricName.named("namespace", "some-counter"));
    counter.inc(1);
    counter.inc(41);
    counter.inc(1);
    counter.inc(-1);
    WorkItemStatus status = new WorkItemStatus();
    when(worker.getOutputCounters()).thenReturn(counterSet);
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedCounter, expectedMetric));
}

Example 9 with CounterMetadata

use of com.google.api.services.dataflow.model.CounterMetadata in project beam by apache.

the class DataflowCounterUpdateExtractor method getStructuredName.

private CounterStructuredNameAndMetadata getStructuredName(CounterName name, String kind) {
    CounterMetadata metadata = new CounterMetadata();
    metadata.setKind(kind);
    CounterStructuredName structuredName = new CounterStructuredName();
    structuredName.setName(name.name());
    if (name.usesContextOriginalName()) {
        structuredName.setOriginalStepName(name.contextOriginalName());
    } else if (name.usesContextSystemName()) {
        structuredName.setComponentStepName(name.contextSystemName());
    }
    if (name.originalRequestingStepName() != null) {
        structuredName.setOriginalRequestingStepName(name.originalRequestingStepName());
    }
    if (name.inputIndex() != null && name.inputIndex() > 0) {
        structuredName.setInputIndex(name.inputIndex());
    }
    CounterStructuredNameAndMetadata nameAndMetadata = new CounterStructuredNameAndMetadata();
    nameAndMetadata.setMetadata(metadata);
    nameAndMetadata.setName(structuredName);
    return nameAndMetadata;
}

Example 10 with CounterMetadata

use of com.google.api.services.dataflow.model.CounterMetadata in project beam by apache.

the class UserMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Transforms user counter MonitoringInfo to relevant CounterUpdate.
 *
 * @return Relevant CounterUpdate or null if transformation failed.
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    long value = decodeInt64Counter(monitoringInfo.getPayload());
    Map<String, String> miLabels = monitoringInfo.getLabelsMap();
    final String ptransform = miLabels.get(MonitoringInfoConstants.Labels.PTRANSFORM);
    final String counterName = miLabels.get(MonitoringInfoConstants.Labels.NAME);
    final String counterNamespace = miLabels.get(MonitoringInfoConstants.Labels.NAMESPACE);
    CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
    DataflowStepContext stepContext = transformIdMapping.get(ptransform);
    name.setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setOriginNamespace(counterNamespace)).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(value));
}