Examples with CounterUpdate com.google.api.services.dataflow.model.CounterUpdate used on opensource projects

Example 21 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class UserDistributionMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived.

@Test
public void testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived() throws Exception {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    NameContext nc = NameContext.create("anyStageName", "anyOriginalName", "anySystemName", "anyUserName");
    DataflowStepContext dsc = mock(DataflowStepContext.class);
    when(dsc.getNameContext()).thenReturn(nc);
    stepContextMapping.put("anyValue", dsc);
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.USER_DISTRIBUTION_INT64).setType(TypeUrns.DISTRIBUTION_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.NAME, "anyName").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyNamespace").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").setPayload(encodeInt64Distribution(DistributionData.create(2L, /* sum */
    1L, /* count */
    3L, /* min */
    4L))).build();
    UserDistributionMonitoringInfoToCounterUpdateTransformer testObject = new UserDistributionMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    CounterUpdate result = testObject.transform(monitoringInfo);
    assertNotNull(result);
    assertEqualsAsJson("{cumulative:true, distribution:{count:{highBits:0, lowBits:1}, " + "max:{highBits:0, lowBits:4}, min:{highBits:0, lowBits:3}, " + "sum:{highBits:0, lowBits:2}}, " + "structuredNameAndMetadata:{metadata:{kind:'DISTRIBUTION'}, " + "name:{name:'anyName', origin:'USER', originNamespace:'anyNamespace', " + "originalStepName:'anyOriginalName'}}}", result);
}

Example 22 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMsecCounter.

@Test
public void populateCounterUpdatesWithMsecCounter() throws Exception {
    final CounterUpdate expectedMsec = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("SYSTEM").setName("start-msecs").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of());
    when(context.extractMsecCounters(anyBoolean())).thenReturn(ImmutableList.of(expectedMsec));
    WorkItemStatus status = new WorkItemStatus();
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedMsec));
}

Example 23 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithOutputCounters.

@Test
public /**
 * Validates that an "internal" Counter is reported.
 */
void populateCounterUpdatesWithOutputCounters() throws Exception {
    final CounterUpdate counter = new CounterUpdate().setNameAndKind(new NameAndKind().setName("some-counter").setKind(Kind.SUM.toString())).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    CounterSet counterSet = new CounterSet();
    counterSet.intSum(CounterName.named("some-counter")).addValue(42);
    WorkItemStatus status = new WorkItemStatus();
    when(worker.getOutputCounters()).thenReturn(counterSet);
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, executionContext);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(counter));
}

Example 24 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class WorkItemStatusClientTest method populateCounterUpdatesWithMetricsAndCounters.

/**
 * Validates that Beam Metrics and "internal" Counters are merged in the update.
 */
@Test
public void populateCounterUpdatesWithMetricsAndCounters() throws Exception {
    final CounterUpdate expectedCounter = new CounterUpdate().setNameAndKind(new NameAndKind().setName("some-counter").setKind(Kind.SUM.toString())).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    CounterSet counterSet = new CounterSet();
    counterSet.intSum(CounterName.named("some-counter")).addValue(42);
    final CounterUpdate expectedMetric = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-counter").setOriginalStepName("step")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(42));
    MetricsContainerImpl metricsContainer = new MetricsContainerImpl("step");
    BatchModeExecutionContext context = mock(BatchModeExecutionContext.class);
    when(context.extractMetricUpdates(anyBoolean())).thenReturn(ImmutableList.of(expectedMetric));
    when(context.extractMsecCounters(anyBoolean())).thenReturn(Collections.emptyList());
    CounterCell counter = metricsContainer.getCounter(MetricName.named("namespace", "some-counter"));
    counter.inc(1);
    counter.inc(41);
    counter.inc(1);
    counter.inc(-1);
    WorkItemStatus status = new WorkItemStatus();
    when(worker.getOutputCounters()).thenReturn(counterSet);
    when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList());
    statusClient.setWorker(worker, context);
    statusClient.populateCounterUpdates(status);
    assertThat(status.getCounterUpdates(), containsInAnyOrder(expectedCounter, expectedMetric));
}

Example 25 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterUpdateAggregators method aggregate.

/**
 * Try to aggregate a List of CounterUpdates. The first CounterUpdate entry of the List will be
 * examined to identify the CounterUpdate kind with {@link #getCounterUpdateKind(CounterUpdate)}
 * and find the suitable {@link CounterUpdateAggregator}, if there is no suitable aggregator the
 * original list will be returned.
 *
 * <p>Note that this method assumes the CounterUpdate elements in this list has the same {@link
 * com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata
 * StructruredNameAndMetadata} or {@link com.google.api.services.dataflow.model.NameAndKind
 * NameAndKind}, also the value type should be the same across all the elements.
 *
 * @param counterUpdates List of CounterUpdate to be aggregated.
 * @return A singleton list of combined CounterUpdate if it is possible to aggregate the elements,
 *     other wise return the original list.
 */
public static List<CounterUpdate> aggregate(List<CounterUpdate> counterUpdates) {
    if (counterUpdates == null || counterUpdates.isEmpty()) {
        return counterUpdates;
    }
    CounterUpdate first = counterUpdates.get(0);
    String kind = getCounterUpdateKind(first);
    if (aggregators.containsKey(kind)) {
        // Return list containing combined CounterUpdate
        return Collections.singletonList(aggregators.get(kind).aggregate(counterUpdates));
    }
    // not able to aggregate the counter updates.
    return counterUpdates;
}