Examples with Storage com.google.api.services.storage.Storage used on opensource projects

Example 1 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testAccessDeniedObjectThrowsIOException.

// GCSUtil.expand() should fail for other errors such as access denied.
@Test
public void testAccessDeniedObjectThrowsIOException() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/accessdeniedfile");
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(expectedException);
    thrown.expect(IOException.class);
    thrown.expectMessage("Unable to get the file object for path");
    gcsUtil.expand(pattern);
}

Example 2 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testNonExistentObjectReturnsEmptyResult.

// GCSUtil.expand() should fail when matching a single object when that object does not exist.
// We should return the empty result since GCS get object is strongly consistent.
@Test
public void testNonExistentObjectReturnsEmptyResult() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile");
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see");
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(expectedException);
    assertEquals(Collections.EMPTY_LIST, gcsUtil.expand(pattern));
}

Example 3 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class RetryHttpRequestInitializerTest method testIOExceptionHandlerIsInvokedOnTimeout.

/**
   * Tests that when RPCs fail with {@link SocketTimeoutException}, the IO exception handler
   * is invoked.
   */
@Test
public void testIOExceptionHandlerIsInvokedOnTimeout() throws Exception {
    // Counts the number of calls to execute the HTTP request.
    final AtomicLong executeCount = new AtomicLong();
    // 10 is a private internal constant in the Google API Client library. See
    // com.google.api.client.http.HttpRequest#setNumberOfRetries
    // TODO: update this test once the private internal constant is public.
    final int defaultNumberOfRetries = 10;
    // A mock HTTP request that always throws SocketTimeoutException.
    MockHttpTransport transport = new MockHttpTransport.Builder().setLowLevelHttpRequest(new MockLowLevelHttpRequest() {

        @Override
        public LowLevelHttpResponse execute() throws IOException {
            executeCount.incrementAndGet();
            throw new SocketTimeoutException("Fake forced timeout exception");
        }
    }).build();
    // A sample HTTP request to Google Cloud Storage that uses both default Transport and default
    // RetryHttpInitializer.
    Storage storage = new Storage.Builder(transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer()).build();
    Get getRequest = storage.objects().get("gs://fake", "file");
    try {
        getRequest.execute();
        fail();
    } catch (Throwable e) {
        assertThat(e, Matchers.<Throwable>instanceOf(SocketTimeoutException.class));
        assertEquals(1 + defaultNumberOfRetries, executeCount.get());
    }
}

Example 4 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testCreateBucketAccessErrors.

@Test
public void testCreateBucketAccessErrors() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.insert(any(String.class), any(Bucket.class))).thenReturn(mockStorageInsert);
    when(mockStorageInsert.execute()).thenThrow(expectedException);
    thrown.expect(AccessDeniedException.class);
    gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}

Example 5 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testGetSizeBytesWhenFileNotFoundBatch.

@Test
public void testGetSizeBytesWhenFileNotFoundBatch() throws Exception {
    JsonFactory jsonFactory = new JacksonFactory();
    String contentBoundary = "batch_foobarbaz";
    String contentBoundaryLine = "--" + contentBoundary;
    String endOfContentBoundaryLine = "--" + contentBoundary + "--";
    GenericJson error = new GenericJson().set("error", new GenericJson().set("code", 404));
    error.setFactory(jsonFactory);
    String content = contentBoundaryLine + "\n" + "Content-Type: application/http\n" + "\n" + "HTTP/1.1 404 Not Found\n" + "Content-Length: -1\n" + "\n" + error.toString() + "\n" + "\n" + endOfContentBoundaryLine + "\n";
    thrown.expect(FileNotFoundException.class);
    MockLowLevelHttpResponse notFoundResponse = new MockLowLevelHttpResponse().setContentType("multipart/mixed; boundary=" + contentBoundary).setContent(content).setStatusCode(HttpStatusCodes.STATUS_CODE_OK);
    MockHttpTransport mockTransport = new MockHttpTransport.Builder().setLowLevelHttpResponse(notFoundResponse).build();
    GcsUtil gcsUtil = gcsOptionsWithTestCredential().getGcsUtil();
    gcsUtil.setStorageClient(new Storage(mockTransport, Transport.getJsonFactory(), null));
    gcsUtil.fileSizes(ImmutableList.of(GcsPath.fromComponents("testbucket", "testobject")));
}