Examples with GcsOptions org.apache.beam.sdk.extensions.gcp.options.GcsOptions used on opensource projects

Example 21 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testUploadBufferSizeUserSpecified.

@Test
public void testUploadBufferSizeUserSpecified() {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    pipelineOptions.setGcsUploadBufferSizeBytes(12345);
    GcsUtil util = pipelineOptions.getGcsUtil();
    assertEquals((Integer) 12345, util.getUploadBufferSizeBytes());
}

Example 22 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testFileSizeNonBatch.

@Test
public void testFileSizeNonBatch() throws Exception {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));
    assertEquals(1000, gcsUtil.fileSize(GcsPath.fromComponents("testbucket", "testobject")));
}

Example 23 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testRetryFileSizeNonBatch.

@Test
public void testRetryFileSizeNonBatch() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    Storage mockStorage = Mockito.mock(Storage.class);
    gcsUtil.setStorageClient(mockStorage);
    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff());
    when(mockStorage.objects()).thenReturn(mockStorageObjects);
    when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
    when(mockStorageGet.execute()).thenThrow(new SocketTimeoutException("SocketException")).thenThrow(new SocketTimeoutException("SocketException")).thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));
    assertEquals(1000, gcsUtil.getObject(GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()).getSize().longValue());
    assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis());
}

Example 24 with GcsOptions

use of org.apache.beam.sdk.extensions.gcp.options.GcsOptions in project beam by apache.

the class GcsUtilTest method testCreationWithDefaultOptions.

@Test
public void testCreationWithDefaultOptions() {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    assertNotNull(pipelineOptions.getGcpCredential());
}