Examples with BackOff com.google.api.client.util.BackOff used on opensource projects

Example 1 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class BigQueryTableRowIterator method executeWithBackOff.

// Execute a BQ request with exponential backoff and return the result.
// client - BQ request to be executed
// error - Formatted message to log if when a request fails. Takes exception message as a
// formatter parameter.
private static <T> T executeWithBackOff(AbstractGoogleClientRequest<T> client, String error) throws IOException, InterruptedException {
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff());
    T result = null;
    while (true) {
        try {
            result = client.execute();
            break;
        } catch (IOException e) {
            LOG.error("{}", error, e);
            if (!BackOffUtils.next(sleeper, backOff)) {
                String errorMessage = String.format("%s Failing to execute job after %d attempts.", error, MAX_RETRIES + 1);
                LOG.error("{}", errorMessage, e);
                throw new IOException(errorMessage, e);
            }
        }
    }
    return result;
}

Example 2 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class FakeJobService method pollJob.

@Override
public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
    BackOff backoff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(maxAttempts).withInitialBackoff(Duration.millis(10)).withMaxBackoff(Duration.standardSeconds(1)).backoff());
    Sleeper sleeper = Sleeper.DEFAULT;
    try {
        do {
            Job job = getJob(jobRef);
            if (job != null) {
                JobStatus status = job.getStatus();
                if (status != null && status.getState() != null && (status.getState().equals("DONE") || status.getState().equals("FAILED"))) {
                    return job;
                }
            }
        } while (BackOffUtils.next(sleeper, backoff));
    } catch (IOException e) {
        return null;
    }
    return null;
}

Example 3 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class DataflowPipelineJob method waitUntilFinish.

/**
   * Waits until the pipeline finishes and returns the final status.
   *
   * @param duration The time to wait for the job to finish.
   *     Provide a value less than 1 ms for an infinite wait.
   *
   * @param messageHandler If non null this handler will be invoked for each
   *   batch of messages received.
   * @param sleeper A sleeper to use to sleep between attempts.
   * @param nanoClock A nanoClock used to time the total time taken.
   * @return The final state of the job or null on timeout.
   * @throws IOException If there is a persistent problem getting job
   *   information.
   * @throws InterruptedException if the thread is interrupted.
   */
@Nullable
@VisibleForTesting
State waitUntilFinish(Duration duration, @Nullable MonitoringUtil.JobMessagesHandler messageHandler, Sleeper sleeper, NanoClock nanoClock, MonitoringUtil monitor) throws IOException, InterruptedException {
    BackOff backoff;
    if (!duration.isLongerThan(Duration.ZERO)) {
        backoff = BackOffAdapter.toGcpBackOff(MESSAGES_BACKOFF_FACTORY.backoff());
    } else {
        backoff = BackOffAdapter.toGcpBackOff(MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(duration).backoff());
    }
    // This function tracks the cumulative time from the *first request* to enforce the wall-clock
    // limit. Any backoff instance could, at best, track the the time since the first attempt at a
    // given request. Thus, we need to track the cumulative time ourselves.
    long startNanos = nanoClock.nanoTime();
    State state;
    do {
        // Get the state of the job before listing messages. This ensures we always fetch job
        // messages after the job finishes to ensure we have all them.
        state = getStateWithRetries(BackOffAdapter.toGcpBackOff(STATUS_BACKOFF_FACTORY.withMaxRetries(0).backoff()), sleeper);
        boolean hasError = state == State.UNKNOWN;
        if (messageHandler != null && !hasError) {
            // Process all the job messages that have accumulated so far.
            try {
                List<JobMessage> allMessages = monitor.getJobMessages(jobId, lastTimestamp);
                if (!allMessages.isEmpty()) {
                    lastTimestamp = fromCloudTime(allMessages.get(allMessages.size() - 1).getTime()).getMillis();
                    messageHandler.process(allMessages);
                }
            } catch (GoogleJsonResponseException | SocketTimeoutException e) {
                hasError = true;
                LOG.warn("There were problems getting current job messages: {}.", e.getMessage());
                LOG.debug("Exception information:", e);
            }
        }
        if (!hasError) {
            // We can stop if the job is done.
            if (state.isTerminal()) {
                switch(state) {
                    case DONE:
                    case CANCELLED:
                        LOG.info("Job {} finished with status {}.", getJobId(), state);
                        break;
                    case UPDATED:
                        LOG.info("Job {} has been updated and is running as the new job with id {}. " + "To access the updated job on the Dataflow monitoring console, " + "please navigate to {}", getJobId(), getReplacedByJob().getJobId(), MonitoringUtil.getJobMonitoringPageURL(getReplacedByJob().getProjectId(), getReplacedByJob().getJobId()));
                        break;
                    default:
                        LOG.info("Job {} failed with status {}.", getJobId(), state);
                }
                return state;
            }
            // The job is not done, so we must keep polling.
            backoff.reset();
            // allotted time.
            if (duration.isLongerThan(Duration.ZERO)) {
                long nanosConsumed = nanoClock.nanoTime() - startNanos;
                Duration consumed = Duration.millis((nanosConsumed + 999999) / 1000000);
                Duration remaining = duration.minus(consumed);
                if (remaining.isLongerThan(Duration.ZERO)) {
                    backoff = BackOffAdapter.toGcpBackOff(MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(remaining).backoff());
                } else {
                    // If there is no time remaining, don't bother backing off.
                    backoff = BackOff.STOP_BACKOFF;
                }
            }
        }
    } while (BackOffUtils.next(sleeper, backoff));
    LOG.warn("No terminal state was returned. State value {}", state);
    // Timed out.
    return null;
}

Example 4 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class PackageUtil method stageOnePackage.

/**
   * Utility to verify whether a package has already been staged and, if not, copy it to the
   * staging location.
   */
private static void stageOnePackage(PackageAttributes attributes, AtomicInteger numUploaded, AtomicInteger numCached, Sleeper retrySleeper, CreateOptions createOptions) {
    String source = attributes.getSourcePath();
    String target = attributes.getDataflowPackage().getLocation();
    // always using MimeTypes.BINARY?
    try {
        try {
            long remoteLength = FileSystems.matchSingleFileSpec(target).sizeBytes();
            if (remoteLength == attributes.getSize()) {
                LOG.debug("Skipping classpath element already staged: {} at {}", attributes.getSourcePath(), target);
                numCached.incrementAndGet();
                return;
            }
        } catch (FileNotFoundException expected) {
        // If the file doesn't exist, it means we need to upload it.
        }
        // Upload file, retrying on failure.
        BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
        while (true) {
            try {
                LOG.debug("Uploading classpath element {} to {}", source, target);
                try (WritableByteChannel writer = makeWriter(target, createOptions)) {
                    copyContent(source, writer);
                }
                numUploaded.incrementAndGet();
                break;
            } catch (IOException e) {
                if (ERROR_EXTRACTOR.accessDenied(e)) {
                    String errorMessage = String.format("Uploaded failed due to permissions error, will NOT retry staging " + "of classpath %s. Please verify credentials are valid and that you have " + "write access to %s. Stale credentials can be resolved by executing " + "'gcloud auth application-default login'.", source, target);
                    LOG.error(errorMessage);
                    throw new IOException(errorMessage, e);
                }
                long sleep = backoff.nextBackOffMillis();
                if (sleep == BackOff.STOP) {
                    // Rethrow last error, to be included as a cause in the catch below.
                    LOG.error("Upload failed, will NOT retry staging of classpath: {}", source, e);
                    throw e;
                } else {
                    LOG.warn("Upload attempt failed, sleeping before retrying staging of classpath: {}", source, e);
                    retrySleeper.sleep(sleep);
                }
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("Could not stage classpath element: " + source, e);
    }
}

Example 5 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class GcsUtilTest method testCreateBucketAccessErrors.

@Test
public void testCreateBucketAccessErrors() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.insert(any(String.class), any(Bucket.class))).thenReturn(mockStorageInsert);
    when(mockStorageInsert.execute()).thenThrow(expectedException);
    thrown.expect(AccessDeniedException.class);
    gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}