Examples with CounterUpdate com.google.api.services.dataflow.model.CounterUpdate used on opensource projects

Example 31 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class ElementCountMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Generates CounterUpdate to send to DFE based on ElementCount MonitoringInfo.
 *
 * @param monitoringInfo Monitoring info to transform.
 * @return CounterUpdate generated based on provided monitoringInfo
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    long value = decodeInt64Counter(monitoringInfo.getPayload());
    final String pcollectionId = monitoringInfo.getLabelsMap().get(MonitoringInfoConstants.Labels.PCOLLECTION);
    final String pcollectionName = pcollectionIdToNameContext.get(pcollectionId).userName();
    String counterName = pcollectionName + "-ElementCount";
    NameAndKind name = new NameAndKind();
    name.setName(counterName).setKind(Kind.SUM.toString());
    return new CounterUpdate().setNameAndKind(name).setCumulative(true).setInteger(DataflowCounterUpdateExtractor.longToSplitInt(value));
}

Example 32 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class DataflowCounterUpdateExtractor method initUpdate.

private CounterUpdate initUpdate(CounterName name, boolean delta, String kind) {
    CounterUpdate counterUpdate = new CounterUpdate();
    if (name.isStructured()) {
        counterUpdate.setStructuredNameAndMetadata(getStructuredName(name, kind));
    } else {
        counterUpdate.setNameAndKind(getUnstructuredName(name, kind));
    }
    counterUpdate.setCumulative(!delta);
    return counterUpdate;
}

Example 33 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class StreamingDataflowWorker method sendWorkerUpdatesToDataflowService.

/**
 * Sends counter updates to Dataflow backend.
 */
private void sendWorkerUpdatesToDataflowService(CounterSet deltaCounters, CounterSet cumulativeCounters) throws IOException {
    // Throttle time is tracked by the windmillServer but is reported to DFE here.
    windmillQuotaThrottling.addValue(windmillServer.getAndResetThrottleTime());
    if (memoryMonitor.isThrashing()) {
        memoryThrashing.addValue(1);
    }
    List<CounterUpdate> counterUpdates = new ArrayList<>(128);
    if (publishCounters) {
        stageInfoMap.values().forEach(s -> counterUpdates.addAll(s.extractCounterUpdates()));
        counterUpdates.addAll(cumulativeCounters.extractUpdates(false, DataflowCounterUpdateExtractor.INSTANCE));
        counterUpdates.addAll(deltaCounters.extractModifiedDeltaUpdates(DataflowCounterUpdateExtractor.INSTANCE));
        if (hasExperiment(options, "beam_fn_api")) {
            Map<Object, List<CounterUpdate>> fnApiCounters = new HashMap<>();
            while (!this.pendingMonitoringInfos.isEmpty()) {
                final CounterUpdate item = this.pendingMonitoringInfos.poll();
                // WorkItem.
                if (item.getCumulative()) {
                    item.setCumulative(false);
                    // Group counterUpdates by counterUpdateKey so they can be aggregated before sending to
                    // dataflow service.
                    fnApiCounters.computeIfAbsent(getCounterUpdateKey(item), k -> new ArrayList<>()).add(item);
                } else {
                    // This is a safety check in case new counter type appears in FnAPI.
                    throw new UnsupportedOperationException("FnApi counters are expected to provide cumulative values." + " Please, update conversion to delta logic" + " if non-cumulative counter type is required.");
                }
            }
            // so we can avoid excessive I/Os for reporting to dataflow service.
            for (List<CounterUpdate> counterUpdateList : fnApiCounters.values()) {
                if (counterUpdateList.isEmpty()) {
                    continue;
                }
                List<CounterUpdate> aggregatedCounterUpdateList = CounterUpdateAggregators.aggregate(counterUpdateList);
                // updates.
                if (aggregatedCounterUpdateList.size() > 10) {
                    CounterUpdate head = aggregatedCounterUpdateList.get(0);
                    this.counterAggregationErrorCount.getAndIncrement();
                    // log warning message only when error count is the power of 2 to avoid spamming.
                    if (this.counterAggregationErrorCount.get() > 10 && Long.bitCount(this.counterAggregationErrorCount.get()) == 1) {
                        LOG.warn("Found non-aggregated counter updates of size {} with kind {}, this will likely " + "cause performance degradation and excessive GC if size is large.", counterUpdateList.size(), MoreObjects.firstNonNull(head.getNameAndKind(), head.getStructuredNameAndMetadata()));
                    }
                }
                counterUpdates.addAll(aggregatedCounterUpdateList);
            }
        }
    }
    // Handle duplicate counters from different stages. Store all the counters in a multi-map and
    // send the counters that appear multiple times in separate RPCs. Same logical counter could
    // appear in multiple stages if a step runs in multiple stages (as with flatten-unzipped stages)
    // especially if the counter definition does not set execution_step_name.
    ListMultimap<Object, CounterUpdate> counterMultimap = MultimapBuilder.hashKeys(counterUpdates.size()).linkedListValues().build();
    boolean hasDuplicates = false;
    for (CounterUpdate c : counterUpdates) {
        Object key = getCounterUpdateKey(c);
        if (counterMultimap.containsKey(key)) {
            hasDuplicates = true;
        }
        counterMultimap.put(key, c);
    }
    // Clears counterUpdates and enqueues unique counters from counterMultimap. If a counter
    // appears more than once, one of them is extracted leaving the remaining in the map.
    Runnable extractUniqueCounters = () -> {
        counterUpdates.clear();
        for (Iterator<Object> iter = counterMultimap.keySet().iterator(); iter.hasNext(); ) {
            List<CounterUpdate> counters = counterMultimap.get(iter.next());
            counterUpdates.add(counters.get(0));
            if (counters.size() == 1) {
                // There is single value. Remove the entry through the iterator.
                iter.remove();
            } else {
                // Otherwise remove the first value.
                counters.remove(0);
            }
        }
    };
    if (hasDuplicates) {
        extractUniqueCounters.run();
    } else {
        // Common case: no duplicates. We can just send counterUpdates, empty the multimap.
        counterMultimap.clear();
    }
    List<Status> errors;
    synchronized (pendingFailuresToReport) {
        errors = new ArrayList<>(pendingFailuresToReport.size());
        for (String stackTrace : pendingFailuresToReport) {
            errors.add(new Status().setCode(// rpc.Code.UNKNOWN
            2).setMessage(stackTrace));
        }
        // Best effort only, no need to wait till successfully sent.
        pendingFailuresToReport.clear();
    }
    WorkItemStatus workItemStatus = new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setErrors(errors).setCounterUpdates(counterUpdates);
    workUnitClient.reportWorkItemStatus(workItemStatus);
    // Send any counters appearing more than once in subsequent RPCs:
    while (!counterMultimap.isEmpty()) {
        extractUniqueCounters.run();
        workUnitClient.reportWorkItemStatus(new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setCounterUpdates(counterUpdates));
    }
}

Example 34 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class MetricsToCounterUpdateConverter method fromDistribution.

public static CounterUpdate fromDistribution(MetricKey key, boolean isCumulative, DistributionData update) {
    CounterStructuredNameAndMetadata name = structuredNameAndMetadata(key, Kind.DISTRIBUTION);
    DistributionUpdate distributionUpdateProto = new DistributionUpdate();
    distributionUpdateProto.setMin(longToSplitInt(update.min())).setMax(longToSplitInt(update.max())).setCount(longToSplitInt(update.count())).setSum(longToSplitInt(update.sum()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(isCumulative).setDistribution(distributionUpdateProto);
}

Example 35 with CounterUpdate

use of com.google.api.services.dataflow.model.CounterUpdate in project beam by apache.

the class CounterShortIdCacheTest method testCacheStructuredName.

@Test
public void testCacheStructuredName() {
    CounterShortIdCache shortIdCache = new CounterShortIdCache();
    ReportWorkItemStatusRequest request = new ReportWorkItemStatusRequest();
    ReportWorkItemStatusResponse reply = new ReportWorkItemStatusResponse();
    // setup mock counters, three work statuses, one with two counters, one with one, one with none
    request.setWorkItemStatuses(createWorkStatusStructuredName(new String[] { "counter", "counter1" }, new String[] {}, new String[] { "counter2" }));
    reply.setWorkItemServiceStates(createWorkServiceState(new Long[] { 1000L, 1001L }, new Long[] {}, new Long[] { 1002L }));
    // Verify the empty case
    WorkItemStatus status1 = request.getWorkItemStatuses().get(0);
    WorkItemStatus status2 = request.getWorkItemStatuses().get(1);
    WorkItemStatus status3 = request.getWorkItemStatuses().get(2);
    shortIdCache.shortenIdsIfAvailable(status1.getCounterUpdates());
    for (CounterUpdate update : status1.getCounterUpdates()) {
        assertNull(update.getShortId());
    }
    // Add the shortIds
    shortIdCache.storeNewShortIds(request, reply);
    shortIdCache.shortenIdsIfAvailable(status1.getCounterUpdates());
    shortIdCache.shortenIdsIfAvailable(status2.getCounterUpdates());
    shortIdCache.shortenIdsIfAvailable(status3.getCounterUpdates());
    checkStatusAndShortIds(status1, 1000L, 1001L);
    checkStatusAndShortIds(status2);
    checkStatusAndShortIds(status3, 1002L);
}