Examples with Storage com.google.api.services.storage.Storage used on opensource projects

Example 16 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testBucketAccessible.

@Test
public void testBucketAccessible() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(new SocketTimeoutException("SocketException")).thenReturn(new Bucket());
    assertTrue(gcsUtil.bucketAccessible(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()));
}

Example 17 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testFileSizeNonBatch.

@Test
public void testFileSizeNonBatch() throws Exception {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));
    assertEquals(1000, gcsUtil.fileSize(GcsPath.fromComponents("testbucket", "testobject")));
}

Example 18 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class RetryHttpRequestInitializerTest method setUp.

@Before
public void setUp() {
    MockitoAnnotations.initMocks(this);
    HttpTransport lowLevelTransport = new HttpTransport() {

        @Override
        protected LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
            return mockLowLevelRequest;
        }
    };
    // Retry initializer will pass through to credential, since we can have
    // only a single HttpRequestInitializer, and we use multiple Credential
    // types in the SDK, not all of which allow for retry configuration.
    RetryHttpRequestInitializer initializer = new RetryHttpRequestInitializer(new MockNanoClock(), new Sleeper() {

        @Override
        public void sleep(long millis) throws InterruptedException {
        }
    }, Arrays.asList(418), mockHttpResponseInterceptor);
    storage = new Storage.Builder(lowLevelTransport, jsonFactory, initializer).setApplicationName("test").build();
}

Example 19 with Storage

use of com.google.api.services.storage.Storage in project beam by apache.

the class GcsUtilTest method testRetryFileSizeNonBatch.

@Test
public void testRetryFileSizeNonBatch() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff());
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(new SocketTimeoutException("SocketException")).thenThrow(new SocketTimeoutException("SocketException")).thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));
    assertEquals(1000, gcsUtil.getObject(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()).getSize().longValue());
    assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis());
}

Example 20 with Storage

use of com.google.api.services.storage.Storage in project ignite by apache.

the class TcpDiscoveryGoogleStorageIpFinder method init.

/**
     * Google Cloud Storage initialization.
     *
     * @throws IgniteSpiException In case of error.
     */
private void init() throws IgniteSpiException {
    if (initGuard.compareAndSet(false, true)) {
        if (srvcAccountId == null || srvcAccountP12FilePath == null || projectName == null || bucketName == null) {
            throw new IgniteSpiException("One or more of the required parameters is not set [serviceAccountId=" + srvcAccountId + ", serviceAccountP12FilePath=" + srvcAccountP12FilePath + ", projectName=" + projectName + ", bucketName=" + bucketName + "]");
        }
        try {
            NetHttpTransport httpTransport;
            try {
                httpTransport = GoogleNetHttpTransport.newTrustedTransport();
            } catch (GeneralSecurityException | IOException e) {
                throw new IgniteSpiException(e);
            }
            GoogleCredential cred;
            try {
                cred = new GoogleCredential.Builder().setTransport(httpTransport).setJsonFactory(JacksonFactory.getDefaultInstance()).setServiceAccountId(srvcAccountId).setServiceAccountPrivateKeyFromP12File(new File(srvcAccountP12FilePath)).setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).build();
            } catch (Exception e) {
                throw new IgniteSpiException("Failed to authenticate on Google Cloud Platform", e);
            }
            try {
                storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), cred).setApplicationName(projectName).build();
            } catch (Exception e) {
                throw new IgniteSpiException("Failed to open a storage for given project name: " + projectName, e);
            }
            boolean createBucket = false;
            try {
                Storage.Buckets.Get getBucket = storage.buckets().get(bucketName);
                getBucket.setProjection("full");
                getBucket.execute();
            } catch (GoogleJsonResponseException e) {
                if (e.getStatusCode() == 404) {
                    U.warn(log, "Bucket doesn't exist, will create it [bucketName=" + bucketName + "]");
                    createBucket = true;
                } else
                    throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e);
            } catch (Exception e) {
                throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e);
            }
            if (createBucket) {
                Bucket newBucket = new Bucket();
                newBucket.setName(bucketName);
                try {
                    Storage.Buckets.Insert insertBucket = storage.buckets().insert(projectName, newBucket);
                    insertBucket.setProjection("full");
                    insertBucket.setPredefinedDefaultObjectAcl("projectPrivate");
                    insertBucket.execute();
                } catch (Exception e) {
                    throw new IgniteSpiException("Failed to create the bucket: " + bucketName, e);
                }
            }
        } finally {
            initLatch.countDown();
        }
    } else {
        try {
            U.await(initLatch);
        } catch (IgniteInterruptedCheckedException e) {
            throw new IgniteSpiException("Thread has been interrupted.", e);
        }
        if (storage == null)
            throw new IgniteSpiException("IpFinder has not been initialized properly");
    }
}