Examples with CreateOptions org.apache.beam.sdk.io.fs.CreateOptions used on opensource projects

Example 6 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadFailsWithPermissionsErrorGivesDetailedMessage.

@Test
public void testPackageUploadFailsWithPermissionsErrorGivesDetailedMessage() throws Exception {
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenThrow(new IOException("Failed to write to GCS path " + STAGING_PATH, googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Permission denied", "Test message")));
    try (PackageUtil directPackageUtil = PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
        directPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpFile.getAbsolutePath())), STAGING_PATH, fastNanoClockAndSleeper, createOptions);
        fail("Expected RuntimeException");
    } catch (RuntimeException e) {
        assertThat("Expected RuntimeException wrapping IOException.", e.getCause(), instanceOf(RuntimeException.class));
        assertThat("Expected IOException containing detailed message.", e.getCause().getCause(), instanceOf(IOException.class));
        assertThat(e.getCause().getCause().getMessage(), Matchers.allOf(Matchers.containsString("Uploaded failed due to permissions error"), Matchers.containsString("Stale credentials can be resolved by executing 'gcloud auth application-default " + "login'")));
    } finally {
        verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
        verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
        verifyNoMoreInteractions(mockGcsUtil);
    }
}

Example 7 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadWithFileSucceeds.

@Test
public void testPackageUploadWithFileSucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    String contents = "This is a test!";
    File tmpFile = makeFileWithContents("file.txt", contents);
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenReturn(pipe.sink());
    List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpFile.getAbsolutePath())), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), endsWith(".txt"));
    assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
    assertThat(new LineReader(Channels.newReader(pipe.source(), StandardCharsets.UTF_8.name())).readLine(), equalTo(contents));
}

Example 8 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadWithExplicitPackageName.

@Test
public void testPackageUploadWithExplicitPackageName() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    final String overriddenName = "alias.txt";
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenReturn(pipe.sink());
    List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpFile.getAbsolutePath(), overriddenName)), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), equalTo(overriddenName));
    assertThat(target.getLocation(), RegexMatcher.matches(STAGING_PATH + "alias.txt"));
}

Example 9 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadFailsWhenIOExceptionThrown.

@Test(expected = RuntimeException.class)
public void testPackageUploadFailsWhenIOExceptionThrown() throws Exception {
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenThrow(new IOException("Fake Exception: Upload error"));
    try (PackageUtil directPackageUtil = PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
        directPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpFile.getAbsolutePath())), STAGING_PATH, fastNanoClockAndSleeper, createOptions);
    } finally {
        verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
        verify(mockGcsUtil, times(5)).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
        verifyNoMoreInteractions(mockGcsUtil);
    }
}