Examples with HotKeyDetection com.google.api.services.dataflow.model.HotKeyDetection used on opensource projects

Example 1 with HotKeyDetection

use of com.google.api.services.dataflow.model.HotKeyDetection in project beam by apache.

the class DataflowWorkProgressUpdater method reportProgressHelper.

@Override
protected void reportProgressHelper() throws Exception {
    if (wasAskedToAbort) {
        LOG.info("Service already asked to abort work item, not reporting ignored progress.");
        return;
    }
    WorkItemServiceState result = workItemStatusClient.reportUpdate(dynamicSplitResultToReport, Duration.millis(requestedLeaseDurationMs));
    if (result != null) {
        if (result.getCompleteWorkStatus() != null && result.getCompleteWorkStatus().getCode() != com.google.rpc.Code.OK.getNumber()) {
            LOG.info("Service asked worker to abort with status: {}", result.getCompleteWorkStatus());
            wasAskedToAbort = true;
            worker.abort();
            return;
        }
        if (result.getHotKeyDetection() != null && result.getHotKeyDetection().getUserStepName() != null) {
            HotKeyDetection hotKeyDetection = result.getHotKeyDetection();
            // which is the correct key. The key is also translated into a Java object in the reader.
            if (options.isHotKeyLoggingEnabled()) {
                hotKeyLogger.logHotKeyDetection(hotKeyDetection.getUserStepName(), TimeUtil.fromCloudDuration(hotKeyDetection.getHotKeyAge()), workItemStatusClient.getExecutionContext().getKey());
            } else {
                hotKeyLogger.logHotKeyDetection(hotKeyDetection.getUserStepName(), TimeUtil.fromCloudDuration(hotKeyDetection.getHotKeyAge()));
            }
        }
        // Resets state after a successful progress report.
        dynamicSplitResultToReport = null;
        progressReportIntervalMs = nextProgressReportInterval(fromCloudDuration(result.getReportStatusInterval()).getMillis(), leaseRemainingTime(getLeaseExpirationTimestamp(result)));
        ApproximateSplitRequest suggestedStopPoint = result.getSplitRequest();
        if (suggestedStopPoint != null) {
            LOG.info("Proposing dynamic split of work unit {} at {}", workString(), suggestedStopPoint);
            dynamicSplitResultToReport = worker.requestDynamicSplit(SourceTranslationUtils.toDynamicSplitRequest(suggestedStopPoint));
        }
    }
}

Example 2 with HotKeyDetection

use of com.google.api.services.dataflow.model.HotKeyDetection in project beam by apache.

the class DataflowWorkProgressUpdaterTest method generateServiceState.

private WorkItemServiceState generateServiceState(@Nullable Position suggestedStopPosition, long millisToNextUpdate) {
    WorkItemServiceState responseState = new WorkItemServiceState();
    responseState.setFactory(Transport.getJsonFactory());
    responseState.setLeaseExpireTime(toCloudTime(new Instant(clock.currentTimeMillis() + LEASE_MS)));
    responseState.setReportStatusInterval(toCloudDuration(Duration.millis(millisToNextUpdate)));
    if (suggestedStopPosition != null) {
        responseState.setSplitRequest(ReaderTestUtils.approximateSplitRequestAtPosition(suggestedStopPosition));
    }
    HotKeyDetection hotKeyDetection = new HotKeyDetection();
    hotKeyDetection.setUserStepName(STEP_ID);
    hotKeyDetection.setHotKeyAge(toCloudDuration(HOT_KEY_AGE));
    responseState.setHotKeyDetection(hotKeyDetection);
    return responseState;
}