Examples with CounterStructuredNameAndMetadata com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata used on opensource projects

Example 6 with CounterStructuredNameAndMetadata

use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.

the class DistributionCounterUpdateAggregatorTest method setUp.

@Before
public void setUp() {
    counterUpdates = new ArrayList<>();
    aggregator = new DistributionCounterUpdateAggregator();
    for (int i = 0; i < 10; i++) {
        counterUpdates.add(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.MEAN.toString()))).setDistribution(new DistributionUpdate().setSum(longToSplitInt((long) i)).setMax(longToSplitInt((long) i)).setMin(longToSplitInt((long) i)).setCount(longToSplitInt((long) 1))));
    }
}

Example 7 with CounterStructuredNameAndMetadata

use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.

the class CounterUpdateAggregatorsTest method testAggregateMean.

@Test
public void testAggregateMean() {
    List<CounterUpdate> meanUpdates = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        meanUpdates.add(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.MEAN.toString()))).setIntegerMean(new IntegerMean().setSum(longToSplitInt((long) i)).setCount(longToSplitInt(1L))));
    }
    List<CounterUpdate> aggregated = CounterUpdateAggregators.aggregate(meanUpdates);
    assertEquals(1, aggregated.size());
    CounterUpdate combined = aggregated.get(0);
    assertEquals(45L, splitIntToLong(combined.getIntegerMean().getSum()));
    assertEquals(10L, splitIntToLong(combined.getIntegerMean().getCount()));
}

Example 8 with CounterStructuredNameAndMetadata

use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMsecCounter.

@Test
public void populateCounterUpdatesWithMsecCounter() throws Exception {
    final CounterUpdate expectedMsec = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("SYSTEM").setName("start-msecs").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of());
    when(context.extractMsecCounters(anyBoolean())).thenReturn(ImmutableList.of(expectedMsec));
    WorkItemStatus status = new WorkItemStatus();
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedMsec));
}

Example 9 with CounterStructuredNameAndMetadata

use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMetricsAndCounters.

/**
 * Validates that Beam Metrics and "internal" Counters are merged in the update.
 */
@Test
public void populateCounterUpdatesWithMetricsAndCounters() throws Exception {
    final CounterUpdate expectedCounter = new CounterUpdate().setNameAndKind(new NameAndKind().setName("some-counter").setKind(Kind.SUM.toString())).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    CounterSet counterSet = new CounterSet();
    counterSet.intSum(CounterName.named("some-counter")).addValue(42);
    final CounterUpdate expectedMetric = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-counter").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    MetricsContainerImpl metricsContainer = new MetricsContainerImpl("step");
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of(expectedMetric));
    when(context.extractMsecCounters(anyBoolean())).thenReturn(Collections.emptyList());
    CounterCell counter = metricsContainer.getCounter(MetricName.named("namespace", "some-counter"));
    counter.inc(1);
    counter.inc(41);
    counter.inc(1);
    counter.inc(-1);
    WorkItemStatus status = new WorkItemStatus();
    when(worker.getOutputCounters()).thenReturn(counterSet);
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedCounter, expectedMetric));
}

Example 10 with CounterStructuredNameAndMetadata

use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.

the class DataflowCounterUpdateExtractor method getStructuredName.

private CounterStructuredNameAndMetadata getStructuredName(CounterName name, String kind) {
    CounterMetadata metadata = new CounterMetadata();
    metadata.setKind(kind);
    CounterStructuredName structuredName = new CounterStructuredName();
    structuredName.setName(name.name());
    if (name.usesContextOriginalName()) {
        structuredName.setOriginalStepName(name.contextOriginalName());
    } else if (name.usesContextSystemName()) {
        structuredName.setComponentStepName(name.contextSystemName());
    }
    if (name.originalRequestingStepName() != null) {
        structuredName.setOriginalRequestingStepName(name.originalRequestingStepName());
    }
    if (name.inputIndex() != null && name.inputIndex() > 0) {
        structuredName.setInputIndex(name.inputIndex());
    }
    CounterStructuredNameAndMetadata nameAndMetadata = new CounterStructuredNameAndMetadata();
    nameAndMetadata.setMetadata(metadata);
    nameAndMetadata.setName(structuredName);
    return nameAndMetadata;
}