Examples with CounterUpdate com.google.api.services.dataflow.model.CounterUpdate used on opensource projects

Example 1 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class BatchModeExecutionContextTest method extractMetricUpdatesCounter.

@Test
public void extractMetricUpdatesCounter() {
    BatchModeExecutionContext executionContext = BatchModeExecutionContext.forTesting(PipelineOptionsFactory.create(), "testStage");
    DataflowOperationContext operationContext = executionContext.createOperationContext(NameContextsForTests.nameContextForTest());
    Counter counter = operationContext.metricsContainer().getCounter(MetricName.named("namespace", "some-counter"));
    counter.inc(1);
    counter.inc(41);
    counter.inc(1);
    counter.inc(-1);
    final CounterUpdate expected = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-counter").setOriginalStepName("originalName")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(longToSplitInt(42));
    assertThat(executionContext.extractMetricUpdates(false), containsInAnyOrder(expected));
    executionContext.commitMetricUpdates();
    Counter counterUncommitted = operationContext.metricsContainer().getCounter(MetricName.named("namespace", "uncommitted-counter"));
    counterUncommitted.inc(64);
    final CounterUpdate expectedUncommitted = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("uncommitted-counter").setOriginalStepName("originalName")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(longToSplitInt(64));
    // Expect to get only the uncommitted metric, unless final update.
    assertThat(executionContext.extractMetricUpdates(false), containsInAnyOrder(expectedUncommitted));
    assertThat(executionContext.extractMetricUpdates(true), containsInAnyOrder(expected, expectedUncommitted));
    executionContext.commitMetricUpdates();
    // All Metrics are committed, expect none unless final update.
    assertThat(executionContext.extractMetricUpdates(false), emptyIterable());
    assertThat(executionContext.extractMetricUpdates(true), containsInAnyOrder(expected, expectedUncommitted));
}

Example 2 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterShortIdCacheTest method createMetricUpdateStructuredName.

private CounterUpdate createMetricUpdateStructuredName(final String name) {
    CounterUpdate metricUpdate = new CounterUpdate();
    metricUpdate.setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setName(name)));
    return metricUpdate;
}

Example 3 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterShortIdCacheTest method createWorkStatusStructuredName.

private List<WorkItemStatus> createWorkStatusStructuredName(String[]... counterNames) {
    List<WorkItemStatus> statuses = new ArrayList<>();
    for (String[] names : counterNames) {
        WorkItemStatus status = new WorkItemStatus();
        List<CounterUpdate> counterList = new ArrayList<>();
        for (String name : names) {
            counterList.add(createMetricUpdateStructuredName(name));
        }
        status.setCounterUpdates(counterList);
        statuses.add(status);
    }
    return statuses;
}

Example 4 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterShortIdCacheTest method createWorkStatusNameAndKind.

private List<WorkItemStatus> createWorkStatusNameAndKind(String[]... counterNames) {
    List<WorkItemStatus> statuses = new ArrayList<>();
    for (String[] names : counterNames) {
        WorkItemStatus status = new WorkItemStatus();
        List<CounterUpdate> counterList = new ArrayList<>();
        for (String name : names) {
            counterList.add(createMetricUpdateNameAndKind(name));
        }
        status.setCounterUpdates(counterList);
        statuses.add(status);
    }
    return statuses;
}

Example 5 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterShortIdCacheTest method testCacheNameAndKind.

@Test
public void testCacheNameAndKind() {
    CounterShortIdCache shortIdCache = new CounterShortIdCache();
    ReportWorkItemStatusRequest request = new ReportWorkItemStatusRequest();
    ReportWorkItemStatusResponse reply = new ReportWorkItemStatusResponse();
    // setup mock counters, three work statuses, one with two counters, one with one, one with none
    request.setWorkItemStatuses(createWorkStatusNameAndKind(new String[] { "counter", "counter1" }, new String[] {}, new String[] { "counter2" }));
    reply.setWorkItemServiceStates(createWorkServiceState(new Long[] { 1000L, 1001L }, new Long[] {}, new Long[] { 1002L }));
    // Verify the empty case
    WorkItemStatus status1 = request.getWorkItemStatuses().get(0);
    WorkItemStatus status2 = request.getWorkItemStatuses().get(1);
    WorkItemStatus status3 = request.getWorkItemStatuses().get(2);
    shortIdCache.shortenIdsIfAvailable(status1.getCounterUpdates());
    for (CounterUpdate update : status1.getCounterUpdates()) {
        assertNull(update.getShortId());
    }
    // Add the shortIds
    shortIdCache.storeNewShortIds(request, reply);
    shortIdCache.shortenIdsIfAvailable(status1.getCounterUpdates());
    shortIdCache.shortenIdsIfAvailable(status2.getCounterUpdates());
    shortIdCache.shortenIdsIfAvailable(status3.getCounterUpdates());
    checkStatusAndShortIds(status1, 1000L, 1001L);
    checkStatusAndShortIds(status2);
    checkStatusAndShortIds(status3, 1002L);
}