Examples with GcsOptions org.apache.beam.sdk.extensions.gcp.options.GcsOptions used on opensource projects

Example 1 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class PackageUtilTest method setUp.

@Before
public void setUp() {
    MockitoAnnotations.initMocks(this);
    GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
    pipelineOptions.setGcsUtil(mockGcsUtil);
    FileSystems.setDefaultPipelineOptions(pipelineOptions);
    createOptions = StandardCreateOptions.builder().setMimeType(MimeTypes.BINARY).build();
}

Example 2 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsPathValidatorTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);
    GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class);
    options.setGcpCredential(new TestCredential());
    options.setGcsUtil(mockGcsUtil);
    validator = GcsPathValidator.fromOptions(options);
}

Example 3 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testAccessDeniedObjectThrowsIOException.

// GCSUtil.expand() should fail for other errors such as access denied.
@Test
public void testAccessDeniedObjectThrowsIOException() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/accessdeniedfile");
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for");
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(expectedException);
    thrown.expect(IOException.class);
    thrown.expectMessage("Unable to get the file object for path");
    gcsUtil.expand(pattern);
}

Example 4 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testNonExistentObjectReturnsEmptyResult.

// GCSUtil.expand() should fail when matching a single object when that object does not exist.
// We should return the empty result since GCS get object is strongly consistent.
@Test
public void testNonExistentObjectReturnsEmptyResult() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile");
    GoogleJsonResponseException expectedException = googleJsonResponseException(HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see");
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(expectedException);
    assertEquals(Collections.EMPTY_LIST, gcsUtil.expand(pattern));
}

Example 5 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testCreationWithGcsUtilProvided.

@Test
public void testCreationWithGcsUtilProvided() {
    GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
    GcsUtil gcsUtil = Mockito.mock(GcsUtil.class);
    pipelineOptions.setGcsUtil(gcsUtil);
    assertSame(gcsUtil, pipelineOptions.getGcsUtil());
}