Examples with MatchResult org.apache.beam.sdk.io.fs.MatchResult used on opensource projects

Example 6 with MatchResult

use of org.apache.beam.sdk.io.fs.MatchResult in project beam by apache.

the class GcsFileSystemTest method testMatch.

@Test
public void testMatch() throws Exception {
    Objects modelObjects = new Objects();
    List<StorageObject> items = new ArrayList<>();
    // A directory
    items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));
    // Files within the directory
    items.add(createStorageObject("gs://testbucket/testdirectory/file1name", 1L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file2name", 2L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file3name", 3L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file4name", 4L));
    items.add(createStorageObject("gs://testbucket/testdirectory/otherfile", 5L));
    items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L));
    modelObjects.setItems(items);
    when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class))).thenReturn(modelObjects);
    List<GcsPath> gcsPaths = ImmutableList.of(GcsPath.fromUri("gs://testbucket/testdirectory/non-exist-file"), GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));
    when(mockGcsUtil.getObjects(eq(gcsPaths))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException()), StorageObjectOrIOException.create(createStorageObject("gs://testbucket/testdirectory/otherfile", 4L))));
    List<String> specs = ImmutableList.of("gs://testbucket/testdirectory/file[1-3]*", "gs://testbucket/testdirectory/non-exist-file", "gs://testbucket/testdirectory/otherfile");
    List<MatchResult> matchResults = gcsFileSystem.match(specs);
    assertEquals(3, matchResults.size());
    assertEquals(Status.OK, matchResults.get(0).status());
    assertThat(ImmutableList.of("gs://testbucket/testdirectory/file1name", "gs://testbucket/testdirectory/file2name", "gs://testbucket/testdirectory/file3name"), contains(toFilenames(matchResults.get(0)).toArray()));
    assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
    assertEquals(Status.OK, matchResults.get(2).status());
    assertThat(ImmutableList.of("gs://testbucket/testdirectory/otherfile"), contains(toFilenames(matchResults.get(2)).toArray()));
}

Example 7 with MatchResult

use of org.apache.beam.sdk.io.fs.MatchResult in project beam by apache.

the class GcsFileSystem method matchNonGlobs.

/**
   * Returns {@link MatchResult MatchResults} for the given {@link GcsPath GcsPaths}.
   *
   *<p>The number of returned {@link MatchResult MatchResults} equals to the number of given
   * {@link GcsPath GcsPaths}. Each {@link MatchResult} contains one {@link Metadata}.
   */
@VisibleForTesting
List<MatchResult> matchNonGlobs(List<GcsPath> gcsPaths) throws IOException {
    List<StorageObjectOrIOException> results = options.getGcsUtil().getObjects(gcsPaths);
    ImmutableList.Builder<MatchResult> ret = ImmutableList.builder();
    for (StorageObjectOrIOException result : results) {
        ret.add(toMatchResult(result));
    }
    return ret.build();
}

Example 8 with MatchResult

use of org.apache.beam.sdk.io.fs.MatchResult in project beam by apache.

the class TextIOTest method assertOutputFiles.

public static void assertOutputFiles(String[] elems, final String header, final String footer, int numShards, Path rootLocation, String outputName, String shardNameTemplate) throws Exception {
    List<File> expectedFiles = new ArrayList<>();
    if (numShards == 0) {
        String pattern = rootLocation.toAbsolutePath().resolve(outputName + "*").toString();
        List<MatchResult> matches = FileSystems.match(Collections.singletonList(pattern));
        for (Metadata expectedFile : Iterables.getOnlyElement(matches).metadata()) {
            expectedFiles.add(new File(expectedFile.resourceId().toString()));
        }
    } else {
        for (int i = 0; i < numShards; i++) {
            expectedFiles.add(new File(rootLocation.toString(), DefaultFilenamePolicy.constructName(outputName, shardNameTemplate, "", i, numShards)));
        }
    }
    List<List<String>> actual = new ArrayList<>();
    for (File tmpFile : expectedFiles) {
        try (BufferedReader reader = new BufferedReader(new FileReader(tmpFile))) {
            List<String> currentFile = new ArrayList<>();
            for (; ; ) {
                String line = reader.readLine();
                if (line == null) {
                    break;
                }
                currentFile.add(line);
            }
            actual.add(currentFile);
        }
    }
    List<String> expectedElements = new ArrayList<>(elems.length);
    for (String elem : elems) {
        byte[] encodedElem = CoderUtils.encodeToByteArray(StringUtf8Coder.of(), elem);
        String line = new String(encodedElem);
        expectedElements.add(line);
    }
    List<String> actualElements = Lists.newArrayList(Iterables.concat(FluentIterable.from(actual).transform(removeHeaderAndFooter(header, footer)).toList()));
    assertThat(actualElements, containsInAnyOrder(expectedElements.toArray()));
    assertTrue(Iterables.all(actual, haveProperHeaderAndFooter(header, footer)));
}

Example 9 with MatchResult

use of org.apache.beam.sdk.io.fs.MatchResult in project beam by apache.

the class LocalFileSystemTest method testMatchMultipleWithoutSubdirectoryExpansion.

@Test
public void testMatchMultipleWithoutSubdirectoryExpansion() throws Exception {
    File unmatchedSubDir = temporaryFolder.newFolder("aaa");
    File unmatchedSubDirFile = File.createTempFile("sub-dir-file", "", unmatchedSubDir);
    unmatchedSubDirFile.deleteOnExit();
    List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString(), temporaryFolder.newFile("aa").toString(), temporaryFolder.newFile("ab").toString());
    temporaryFolder.newFile("ba");
    temporaryFolder.newFile("bb");
    List<MatchResult> matchResults = matchGlobWithPathPrefix(temporaryFolder.getRoot().toPath().resolve("a"), "*");
    assertThat(toFilenames(matchResults), containsInAnyOrder(expected.toArray(new String[expected.size()])));
}

Example 10 with MatchResult

use of org.apache.beam.sdk.io.fs.MatchResult in project beam by apache.

the class LocalFileSystemTest method testMatchMultipleWithSubdirectoryExpansion.

@Test
public void testMatchMultipleWithSubdirectoryExpansion() throws Exception {
    File matchedSubDir = temporaryFolder.newFolder("a");
    File matchedSubDirFile = File.createTempFile("sub-dir-file", "", matchedSubDir);
    matchedSubDirFile.deleteOnExit();
    File unmatchedSubDir = temporaryFolder.newFolder("b");
    File unmatchedSubDirFile = File.createTempFile("sub-dir-file", "", unmatchedSubDir);
    unmatchedSubDirFile.deleteOnExit();
    List<String> expected = ImmutableList.of(matchedSubDirFile.toString(), temporaryFolder.newFile("aa").toString(), temporaryFolder.newFile("ab").toString());
    temporaryFolder.newFile("ba");
    temporaryFolder.newFile("bb");
    List<MatchResult> matchResults = matchGlobWithPathPrefix(temporaryFolder.getRoot().toPath().resolve("a"), "**");
    assertThat(toFilenames(matchResults), Matchers.hasItems(expected.toArray(new String[expected.size()])));
}