Examples with DataflowPackage com.google.api.services.dataflow.model.DataflowPackage used on opensource projects

Example 6 with DataflowPackage

use of com.google.api.services.dataflow.model.DataflowPackage in project beam by apache.

the class PackageUtilTest method testStagingPreservesClasspath.

@Test
public void testStagingPreservesClasspath() throws Exception {
    File smallFile = makeFileWithContents("small.txt", "small");
    File largeFile = makeFileWithContents("large.txt", "large contents");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenAnswer(new Answer<SinkChannel>() {

        @Override
        public SinkChannel answer(InvocationOnMock invocation) throws Throwable {
            return Pipe.open().sink();
        }
    });
    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(ImmutableList.of(smallFile.getAbsolutePath(), largeFile.getAbsolutePath()), STAGING_PATH, createOptions);
    // Verify that the packages are returned small, then large, matching input order even though
    // the large file would be uploaded first.
    assertThat(targets.get(0).getName(), startsWith("small"));
    assertThat(targets.get(1).getName(), startsWith("large"));
}

Example 7 with DataflowPackage

use of com.google.api.services.dataflow.model.DataflowPackage in project beam by apache.

the class PackageUtilTest method testPackageNamingWithFilesHavingSameContentsButDifferentNames.

@Test
public void testPackageNamingWithFilesHavingSameContentsButDifferentNames() throws Exception {
    File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
    makeFileWithContents("folder1/folderA/uniqueName1", "This is a test!");
    DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
    File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
    makeFileWithContents("folder2/folderA/uniqueName2", "This is a test!");
    DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
    assertNotEquals(target1.getName(), target2.getName());
    assertNotEquals(target1.getLocation(), target2.getLocation());
}

Example 8 with DataflowPackage

use of com.google.api.services.dataflow.model.DataflowPackage in project beam by apache.

the class PackageUtilTest method testPackageUploadWithEmptyDirectorySucceeds.

@Test
public void testPackageUploadWithEmptyDirectorySucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpDirectory = tmpFolder.newFolder("folder");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
    assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
    assertNull(new ZipInputStream(Channels.newInputStream(pipe.source())).getNextEntry());
}

Example 9 with DataflowPackage

use of com.google.api.services.dataflow.model.DataflowPackage in project beam by apache.

the class PackageUtilTest method testPackageUploadWithExplicitPackageName.

@Test
public void testPackageUploadWithExplicitPackageName() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    final String overriddenName = "alias.txt";
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), equalTo(overriddenName));
    assertThat(target.getLocation(), RegexMatcher.matches(STAGING_PATH + "file-" + HASH_PATTERN + ".txt"));
}

Example 10 with DataflowPackage

use of com.google.api.services.dataflow.model.DataflowPackage in project beam by apache.

the class PackageUtilTest method testPackageUploadWithFileSucceeds.

@Test
public void testPackageUploadWithFileSucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    String contents = "This is a test!";
    File tmpFile = makeFileWithContents("file.txt", contents);
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
    assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
    assertThat(new LineReader(Channels.newReader(pipe.source(), "UTF-8")).readLine(), equalTo(contents));
}