Examples with BackOff com.google.api.client.util.BackOff used on opensource projects

Example 6 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class PackageUtil method stageOnePackage.

/**
   * Utility to verify whether a package has already been staged and, if not, copy it to the
   * staging location.
   */
private static void stageOnePackage(PackageAttributes attributes, AtomicInteger numUploaded, AtomicInteger numCached, Sleeper retrySleeper, CreateOptions createOptions) {
    String source = attributes.getSourcePath();
    String target = attributes.getDataflowPackage().getLocation();
    // always using MimeTypes.BINARY?
    try {
        try {
            long remoteLength = FileSystems.matchSingleFileSpec(target).sizeBytes();
            if (remoteLength == attributes.getSize()) {
                LOG.debug("Skipping classpath element already staged: {} at {}", attributes.getSourcePath(), target);
                numCached.incrementAndGet();
                return;
            }
        } catch (FileNotFoundException expected) {
        // If the file doesn't exist, it means we need to upload it.
        }
        // Upload file, retrying on failure.
        BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
        while (true) {
            try {
                LOG.debug("Uploading classpath element {} to {}", source, target);
                try (WritableByteChannel writer = makeWriter(target, createOptions)) {
                    copyContent(source, writer);
                }
                numUploaded.incrementAndGet();
                break;
            } catch (IOException e) {
                if (ERROR_EXTRACTOR.accessDenied(e)) {
                    String errorMessage = String.format("Uploaded failed due to permissions error, will NOT retry staging " + "of classpath %s. Please verify credentials are valid and that you have " + "write access to %s. Stale credentials can be resolved by executing " + "'gcloud auth application-default login'.", source, target);
                    LOG.error(errorMessage);
                    throw new IOException(errorMessage, e);
                }
                long sleep = backoff.nextBackOffMillis();
                if (sleep == BackOff.STOP) {
                    // Rethrow last error, to be included as a cause in the catch below.
                    LOG.error("Upload failed, will NOT retry staging of classpath: {}", source, e);
                    throw e;
                } else {
                    LOG.warn("Upload attempt failed, sleeping before retrying staging of classpath: {}", source, e);
                    retrySleeper.sleep(sleep);
                }
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("Could not stage classpath element: " + source, e);
    }
}

Example 7 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class GcsUtilTest method testBucketDoesNotExistBecauseOfAccessError.

@Test
public void testBucketDoesNotExistBecauseOfAccessError() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(expectedException);
    assertFalse(gcsUtil.bucketAccessible(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()));
}

Example 8 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class GcsUtilTest method testGetBucket.

@Test
public void testGetBucket() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(new SocketTimeoutException("SocketException")).thenReturn(new Bucket());
    assertNotNull(gcsUtil.getBucket(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()));
}

Example 9 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class GcsUtilTest method testGetBucketNotExists.

@Test
public void testGetBucketNotExists() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(googleJsonResponseException(HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see"));
    thrown.expect(FileNotFoundException.class);
    thrown.expectMessage("It don't exist");
    gcsUtil.getBucket(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper());
}

Example 10 with BackOff

use of com.google.api.client.util.BackOff in project beam by apache.

the class GcsUtilTest method testCreateBucket.

@Test
public void testCreateBucket() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.insert(any(String.class), any(Bucket.class))).thenReturn(mockStorageInsert);
    when(mockStorageInsert.execute()).thenThrow(new SocketTimeoutException("SocketException")).thenReturn(new Bucket());
    gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}