Examples with LocatedFileStatus org.apache.hadoop.fs.LocatedFileStatus used on opensource projects

Example 11 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestDistributedFileSystem method testListFiles.

@Test(timeout = 60000)
public void testListFiles() throws IOException {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        DistributedFileSystem fs = cluster.getFileSystem();
        final Path relative = new Path("relative");
        fs.create(new Path(relative, "foo")).close();
        final List<LocatedFileStatus> retVal = new ArrayList<>();
        final RemoteIterator<LocatedFileStatus> iter = fs.listFiles(relative, true);
        while (iter.hasNext()) {
            retVal.add(iter.next());
        }
        System.out.println("retVal = " + retVal);
    } finally {
        cluster.shutdown();
    }
}

Example 12 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class AbstractContractGetFileStatusTest method testListLocatedStatusFile.

@Test
public void testListLocatedStatusFile() throws Throwable {
    describe("test the listLocatedStatus(path) on a file");
    Path f = touchf("listLocatedStatus");
    List<LocatedFileStatus> statusList = toList(getFileSystem().listLocatedStatus(f));
    assertEquals("size of file list returned", 1, statusList.size());
    assertIsNamedFile(f, statusList.get(0));
    List<LocatedFileStatus> statusList2 = toListThroughNextCallsAlone(getFileSystem().listLocatedStatus(f));
    assertEquals("size of file list returned through next() calls", 1, statusList2.size());
}

Example 13 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class AbstractContractGetFileStatusTest method checkListLocatedStatusStatusComplexDir.

/**
   * Test {@link FileSystem#listLocatedStatus(Path)} on a complex
   * directory tree.
   * @param tree directory tree to list.
   * @throws Throwable
   */
protected void checkListLocatedStatusStatusComplexDir(TreeScanResults tree) throws Throwable {
    describe("Expect listLocatedStatus to list all entries in top dir only");
    FileSystem fs = getFileSystem();
    TreeScanResults listing = new TreeScanResults(fs.listLocatedStatus(tree.getBasePath()));
    listing.assertSizeEquals("listLocatedStatus()", TREE_FILES, TREE_WIDTH, 0);
    verifyFileStats(fs.listLocatedStatus(tree.getBasePath()));
    // listLocatedStatus and listStatus must return the same files.
    TreeScanResults listStatus = new TreeScanResults(fs.listStatus(tree.getBasePath()));
    listing.assertEquivalent(listStatus);
    // now check without using
    List<LocatedFileStatus> statusThroughNext = toListThroughNextCallsAlone(fs.listLocatedStatus(tree.getBasePath()));
    TreeScanResults resultsThroughNext = new TreeScanResults(statusThroughNext);
    listStatus.assertFieldsEquivalent("files", listing, listStatus.getFiles(), resultsThroughNext.getFiles());
}

Example 14 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class AbstractContractGetFileStatusTest method testListLocatedStatusFiltering.

@Test
public void testListLocatedStatusFiltering() throws Throwable {
    describe("Call listLocatedStatus() with filtering");
    describe("Call listStatus() against paths and directories with filtering");
    Path file1 = touchf("file-1.txt");
    Path file2 = touchf("file-2.txt");
    Path parent = file1.getParent();
    FileSystem fs = getFileSystem();
    touch(fs, file1);
    touch(fs, file2);
    // this is not closed: ignore any IDE warnings.
    ExtendedFilterFS xfs = new ExtendedFilterFS(fs);
    List<LocatedFileStatus> result;
    verifyListStatus(0, parent, NO_PATHS);
    verifyListStatus(2, parent, ALL_PATHS);
    MatchesNameFilter file1Filter = new MatchesNameFilter("file-1.txt");
    result = verifyListLocatedStatus(xfs, 1, parent, file1Filter);
    assertEquals(file1, result.get(0).getPath());
    verifyListLocatedStatus(xfs, 0, file1, NO_PATHS);
    verifyListLocatedStatus(xfs, 1, file1, ALL_PATHS);
    assertEquals(file1, result.get(0).getPath());
    verifyListLocatedStatus(xfs, 1, file1, file1Filter);
    assertEquals(file1, result.get(0).getPath());
    verifyListLocatedStatusNextCalls(xfs, 1, file1, file1Filter);
    // empty subdirectory
    Path subdir = path("subdir");
    mkdirs(subdir);
    verifyListLocatedStatus(xfs, 0, subdir, NO_PATHS);
    verifyListLocatedStatus(xfs, 0, subdir, ALL_PATHS);
    verifyListLocatedStatusNextCalls(xfs, 0, subdir, ALL_PATHS);
    verifyListLocatedStatus(xfs, 0, subdir, new MatchesNameFilter("subdir"));
}

Example 15 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestListFilesInFileContext method testDirectory.

/** Test when input path is a directory */
@Test
public void testDirectory() throws IOException {
    fc.mkdir(DIR1, FsPermission.getDefault(), true);
    // test empty directory
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(DIR1, true);
    assertFalse(itor.hasNext());
    itor = fc.util().listFiles(DIR1, false);
    assertFalse(itor.hasNext());
    // testing directory with 1 file
    writeFile(fc, FILE2, FILE_LEN);
    itor = fc.util().listFiles(DIR1, true);
    LocatedFileStatus stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE2), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
    itor = fc.util().listFiles(DIR1, false);
    stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE2), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
    // test more complicated directory
    writeFile(fc, FILE1, FILE_LEN);
    writeFile(fc, FILE3, FILE_LEN);
    itor = fc.util().listFiles(TEST_DIR, true);
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE2), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE3), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
    itor = fc.util().listFiles(TEST_DIR, false);
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
}