Examples with CreateOptions org.apache.beam.sdk.io.fs.CreateOptions used on opensource projects

Example 1 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadWithDirectorySucceeds.

@Test
public void testPackageUploadWithDirectorySucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpDirectory = tmpFolder.newFolder("folder");
    tmpFolder.newFolder("folder", "empty_directory");
    tmpFolder.newFolder("folder", "directory");
    makeFileWithContents("folder/file.txt", "This is a test!");
    makeFileWithContents("folder/directory/file.txt", "This is also a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenReturn(pipe.sink());
    defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpDirectory.getAbsolutePath())), STAGING_PATH, createOptions);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
    verifyNoMoreInteractions(mockGcsUtil);
    List<String> zipEntryNames = new ArrayList<>();
    try (ZipInputStream inputStream = new ZipInputStream(Channels.newInputStream(pipe.source()))) {
        for (ZipEntry entry = inputStream.getNextEntry(); entry != null; entry = inputStream.getNextEntry()) {
            zipEntryNames.add(entry.getName());
        }
    }
    assertThat(zipEntryNames, containsInAnyOrder("directory/file.txt", "empty_directory/", "file.txt"));
}

Example 2 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadIsNotSkippedWhenSizesAreDifferent.

@Test
public void testPackageUploadIsNotSkippedWhenSizesAreDifferent() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpDirectory = tmpFolder.newFolder("folder");
    tmpFolder.newFolder("folder", "empty_directory");
    tmpFolder.newFolder("folder", "directory");
    makeFileWithContents("folder/file.txt", "This is a test!");
    makeFileWithContents("folder/directory/file.txt", "This is also a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(createStorageObject(STAGING_PATH, Long.MAX_VALUE))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenReturn(pipe.sink());
    defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpDirectory.getAbsolutePath())), STAGING_PATH, createOptions);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
    verifyNoMoreInteractions(mockGcsUtil);
}

Example 3 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testStagingPreservesClasspath.

@Test
public void testStagingPreservesClasspath() throws Exception {
    File smallFile = makeFileWithContents("small.txt", "small");
    File largeFile = makeFileWithContents("large.log", "large contents");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenAnswer(invocation -> Pipe.open().sink());
    List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(smallFile.getAbsolutePath()), makeStagedFile(largeFile.getAbsolutePath())), STAGING_PATH, createOptions);
    // Verify that the packages are returned small, then large, matching input order even though
    // the large file would be uploaded first.
    assertThat(targets.get(0).getName(), endsWith(".txt"));
    assertThat(targets.get(1).getName(), endsWith(".log"));
}

Example 4 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadEventuallySucceeds.

@Test
public void testPackageUploadEventuallySucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenThrow(// First attempt fails
    new IOException("Fake Exception: 410 Gone")).thenReturn(// second attempt succeeds
    pipe.sink());
    try (PackageUtil directPackageUtil = PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
        directPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpFile.getAbsolutePath())), STAGING_PATH, fastNanoClockAndSleeper, createOptions);
    } finally {
        verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
        verify(mockGcsUtil, times(2)).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
        verifyNoMoreInteractions(mockGcsUtil);
    }
}

Example 5 with CreateOptions

use of org.apache.beam.sdk.io.fs.CreateOptions in project beam by apache.

the class PackageUtilTest method testPackageUploadWithEmptyDirectorySucceeds.

@Test
public void testPackageUploadWithEmptyDirectorySucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpDirectory = tmpFolder.newFolder("folder");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), any(GcsUtil.CreateOptions.class))).thenReturn(pipe.sink());
    List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements(ImmutableList.of(makeStagedFile(tmpDirectory.getAbsolutePath())), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), any(GcsUtil.CreateOptions.class));
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), endsWith(".jar"));
    assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
    try (ZipInputStream zipInputStream = new ZipInputStream(Channels.newInputStream(pipe.source()))) {
        assertNull(zipInputStream.getNextEntry());
    }
}