Examples with Sleeper com.google.api.client.util.Sleeper used on opensource projects

Example 1 with Sleeper

use of com.google.api.client.util.Sleeper in project beam by apache.

the class BigQueryTableRowIterator method executeWithBackOff.

// Execute a BQ request with exponential backoff and return the result.
// client - BQ request to be executed
// error - Formatted message to log if when a request fails. Takes exception message as a
// formatter parameter.
private static <T> T executeWithBackOff(AbstractGoogleClientRequest<T> client, String error) throws IOException, InterruptedException {
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff());
    T result = null;
    while (true) {
        try {
            result = client.execute();
            break;
        } catch (IOException e) {
            LOG.error("{}", error, e);
            if (!BackOffUtils.next(sleeper, backOff)) {
                String errorMessage = String.format("%s Failing to execute job after %d attempts.", error, MAX_RETRIES + 1);
                LOG.error("{}", errorMessage, e);
                throw new IOException(errorMessage, e);
            }
        }
    }
    return result;
}

Example 2 with Sleeper

use of com.google.api.client.util.Sleeper in project beam by apache.

the class FakeJobService method pollJob.

@Override
public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
    BackOff backoff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(maxAttempts).withInitialBackoff(Duration.millis(10)).withMaxBackoff(Duration.standardSeconds(1)).backoff());
    Sleeper sleeper = Sleeper.DEFAULT;
    try {
        do {
            Job job = getJob(jobRef);
            if (job != null) {
                JobStatus status = job.getStatus();
                if (status != null && status.getState() != null && (status.getState().equals("DONE") || status.getState().equals("FAILED"))) {
                    return job;
                }
            }
        } while (BackOffUtils.next(sleeper, backoff));
    } catch (IOException e) {
        return null;
    }
    return null;
}

Example 3 with Sleeper

use of com.google.api.client.util.Sleeper in project beam by apache.

the class BigQueryServicesImplTest method testStartLoadJobRetry.

/**
   * Tests that {@link BigQueryServicesImpl.JobServiceImpl#startLoadJob} succeeds with a retry.
   */
@Test
public void testStartLoadJobRetry() throws IOException, InterruptedException {
    Job testJob = new Job();
    JobReference jobRef = new JobReference();
    jobRef.setJobId("jobId");
    jobRef.setProjectId("projectId");
    testJob.setJobReference(jobRef);
    // First response is 403 rate limited, second response has valid payload.
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(403).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))).thenReturn(toStream(testJob));
    Sleeper sleeper = new FastNanoClockAndSleeper();
    JobServiceImpl.startJob(testJob, new ApiErrorExtractor(), bigquery, sleeper, BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()));
    verify(response, times(2)).getStatusCode();
    verify(response, times(2)).getContent();
    verify(response, times(2)).getContentType();
}

Example 4 with Sleeper

use of com.google.api.client.util.Sleeper in project beam by apache.

the class BigQueryServicesImplTest method testStartLoadJobSucceeds.

/**
   * Tests that {@link BigQueryServicesImpl.JobServiceImpl#startLoadJob} succeeds.
   */
@Test
public void testStartLoadJobSucceeds() throws IOException, InterruptedException {
    Job testJob = new Job();
    JobReference jobRef = new JobReference();
    jobRef.setJobId("jobId");
    jobRef.setProjectId("projectId");
    testJob.setJobReference(jobRef);
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(testJob));
    Sleeper sleeper = new FastNanoClockAndSleeper();
    JobServiceImpl.startJob(testJob, new ApiErrorExtractor(), bigquery, sleeper, BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()));
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
    expectedLogs.verifyInfo(String.format("Started BigQuery job: %s", jobRef));
}

Example 5 with Sleeper

use of com.google.api.client.util.Sleeper in project beam by apache.

the class DataflowPipelineJobTest method testWaitUntilFinishNoRepeatedLogs.

/**
   * Tests that a {@link DataflowPipelineJob} does not duplicate messages.
   */
@Test
public void testWaitUntilFinishNoRepeatedLogs() throws Exception {
    DataflowPipelineJob job = new DataflowPipelineJob(mockDataflowClient, JOB_ID, options, null);
    Sleeper sleeper = new ZeroSleeper();
    NanoClock nanoClock = mock(NanoClock.class);
    Instant separatingTimestamp = new Instant(42L);
    JobMessage theMessage = infoMessage(separatingTimestamp, "nothing");
    MonitoringUtil mockMonitor = mock(MonitoringUtil.class);
    when(mockMonitor.getJobMessages(anyString(), anyLong())).thenReturn(ImmutableList.of(theMessage));
    // The Job just always reports "running" across all calls
    Job fakeJob = new Job();
    fakeJob.setCurrentState("JOB_STATE_RUNNING");
    when(mockDataflowClient.getJob(anyString())).thenReturn(fakeJob);
    // After waitUntilFinish the DataflowPipelineJob should record the latest message timestamp
    when(nanoClock.nanoTime()).thenReturn(0L).thenReturn(2000000000L);
    job.waitUntilFinish(Duration.standardSeconds(1), mockHandler, sleeper, nanoClock, mockMonitor);
    verify(mockHandler).process(ImmutableList.of(theMessage));
    // Second waitUntilFinish should request jobs with `separatingTimestamp` so the monitor
    // will only return new messages
    when(nanoClock.nanoTime()).thenReturn(3000000000L).thenReturn(6000000000L);
    job.waitUntilFinish(Duration.standardSeconds(1), mockHandler, sleeper, nanoClock, mockMonitor);
    verify(mockMonitor).getJobMessages(anyString(), eq(separatingTimestamp.getMillis()));
}