Examples with GcsOptions org.apache.beam.sdk.extensions.gcp.options.GcsOptions used on opensource projects

Example 6 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testCreateBucketAccessErrors.

@Test
public void testCreateBucketAccessErrors() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.insert(any(String.class), any(Bucket.class))).thenReturn(mockStorageInsert);
    when(mockStorageInsert.execute()).thenThrow(expectedException);
    thrown.expect(AccessDeniedException.class);
    gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}

Example 7 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testMultipleThreadsCanCompleteOutOfOrderWithDefaultThreadPool.

@Test
public void testMultipleThreadsCanCompleteOutOfOrderWithDefaultThreadPool() throws Exception {
    GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
    ExecutorService executorService = pipelineOptions.getExecutorService();
    int numThreads = 100;
    final CountDownLatch[] countDownLatches = new CountDownLatch[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int currentLatch = i;
        countDownLatches[i] = new CountDownLatch(1);
        executorService.execute(new Runnable() {

            @Override
            public void run() {
                // Wait for latch N and then release latch N - 1
                try {
                    countDownLatches[currentLatch].await();
                    if (currentLatch > 0) {
                        countDownLatches[currentLatch - 1].countDown();
                    }
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new RuntimeException(e);
                }
            }
        });
    }
    // Release the last latch starting the chain reaction.
    countDownLatches[countDownLatches.length - 1].countDown();
    executorService.shutdown();
    assertTrue("Expected tasks to complete", executorService.awaitTermination(10, TimeUnit.SECONDS));
}

Example 8 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testUploadBufferSizeDefault.

@Test
public void testUploadBufferSizeDefault() {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil util = pipelineOptions.getGcsUtil();
    assertNull(util.getUploadBufferSizeBytes());
}

Example 9 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testBucketDoesNotExist.

@Test
public void testBucketDoesNotExist() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
    Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
    when(mockStorage.buckets()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(googleJsonResponseException(HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see"));
    assertFalse(gcsUtil.bucketAccessible(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()));
}

Example 10 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testCreationWithExecutorServiceProvided.

@Test
public void testCreationWithExecutorServiceProvided() {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    pipelineOptions.setExecutorService(Executors.newCachedThreadPool());
    assertSame(pipelineOptions.getExecutorService(), pipelineOptions.getGcsUtil().executorService);
}