Examples with LocatedFileStatus org.apache.hadoop.fs.LocatedFileStatus used on opensource projects

Example 56 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class AbstractContractGetFileStatusTest method checkListFilesComplexDirNonRecursive.

/**
   * Test {@link FileSystem#listFiles(Path, boolean)} on a complex
   * directory tree and the recursive flag set to false.
   * @param tree directory tree to list.
   * @throws Throwable
   */
protected void checkListFilesComplexDirNonRecursive(TreeScanResults tree) throws Throwable {
    describe("Expect non-recursive listFiles(false) to list all entries" + " in top dir only");
    FileSystem fs = getFileSystem();
    TreeScanResults listing = new TreeScanResults(fs.listFiles(tree.getBasePath(), false));
    listing.assertSizeEquals("listFiles(false)", TREE_FILES, 0, 0);
    verifyFileStats(fs.listFiles(tree.getBasePath(), false));
    // the files listed should match the set of files in a listStatus() call.
    // the directories are not checked
    TreeScanResults listStatus = new TreeScanResults(fs.listStatus(tree.getBasePath()));
    listStatus.assertFieldsEquivalent("files", listing, listStatus.getFiles(), listing.getFiles());
    List<LocatedFileStatus> statusThroughNext = toListThroughNextCallsAlone(fs.listFiles(tree.getBasePath(), false));
    TreeScanResults resultsThroughNext = new TreeScanResults(statusThroughNext);
    listStatus.assertFieldsEquivalent("files", listing, listStatus.getFiles(), resultsThroughNext.getFiles());
}

Example 57 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class AbstractContractGetFileStatusTest method testListFilesFileRecursive.

@Test
public void testListFilesFileRecursive() throws Throwable {
    describe("test the listFiles(path, true) on a file");
    Path f = touchf("listfilesRecursive");
    List<LocatedFileStatus> statusList = toList(getFileSystem().listFiles(f, true));
    assertEquals("size of file list returned", 1, statusList.size());
    assertIsNamedFile(f, statusList.get(0));
    List<LocatedFileStatus> statusList2 = toListThroughNextCallsAlone(getFileSystem().listFiles(f, true));
    assertEquals("size of file list returned", 1, statusList2.size());
}

Example 58 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class ViewFsBaseTest method testOperationsThroughMountLinks.

/** 
   * Test modify operations (create, mkdir, delete, etc) 
   * on the mount file system where the pathname references through
   * the mount points.  Hence these operation will modify the target
   * file system.
   * 
   * Verify the operation via mountfs (ie fc) and *also* via the
   *  target file system (ie fclocal) that the mount link points-to.
   */
@Test
public void testOperationsThroughMountLinks() throws IOException {
    // Create file 
    fileContextTestHelper.createFileNonRecursive(fcView, "/user/foo");
    Assert.assertTrue("Create file should be file", isFile(fcView, new Path("/user/foo")));
    Assert.assertTrue("Target of created file should be type file", isFile(fcTarget, new Path(targetTestRoot, "user/foo")));
    // Delete the created file
    Assert.assertTrue("Delete should succeed", fcView.delete(new Path("/user/foo"), false));
    Assert.assertFalse("File should not exist after delete", exists(fcView, new Path("/user/foo")));
    Assert.assertFalse("Target File should not exist after delete", exists(fcTarget, new Path(targetTestRoot, "user/foo")));
    // Create file with a 2 component dirs
    fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/linkToDir2/foo");
    Assert.assertTrue("Created file should be type file", isFile(fcView, new Path("/internalDir/linkToDir2/foo")));
    Assert.assertTrue("Target of created file should be type file", isFile(fcTarget, new Path(targetTestRoot, "dir2/foo")));
    // Delete the created file
    Assert.assertTrue("Delete should suceed", fcView.delete(new Path("/internalDir/linkToDir2/foo"), false));
    Assert.assertFalse("File should not exist after deletion", exists(fcView, new Path("/internalDir/linkToDir2/foo")));
    Assert.assertFalse("Target should not exist after deletion", exists(fcTarget, new Path(targetTestRoot, "dir2/foo")));
    // Create file with a 3 component dirs
    fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/internalDir2/linkToDir3/foo");
    Assert.assertTrue("Created file should be of type file", isFile(fcView, new Path("/internalDir/internalDir2/linkToDir3/foo")));
    Assert.assertTrue("Target of created file should also be type file", isFile(fcTarget, new Path(targetTestRoot, "dir3/foo")));
    // Recursive Create file with missing dirs
    fileContextTestHelper.createFile(fcView, "/internalDir/linkToDir2/missingDir/miss2/foo");
    Assert.assertTrue("Created file should be of type file", isFile(fcView, new Path("/internalDir/linkToDir2/missingDir/miss2/foo")));
    Assert.assertTrue("Target of created file should also be type file", isFile(fcTarget, new Path(targetTestRoot, "dir2/missingDir/miss2/foo")));
    // Delete the created file
    Assert.assertTrue("Delete should succeed", fcView.delete(new Path("/internalDir/internalDir2/linkToDir3/foo"), false));
    Assert.assertFalse("Deleted File should not exist", exists(fcView, new Path("/internalDir/internalDir2/linkToDir3/foo")));
    Assert.assertFalse("Target of deleted file should not exist", exists(fcTarget, new Path(targetTestRoot, "dir3/foo")));
    // mkdir
    fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), FileContext.DEFAULT_PERM, false);
    Assert.assertTrue("New dir should be type dir", isDir(fcView, new Path("/user/dirX")));
    Assert.assertTrue("Target of new dir should be of type dir", isDir(fcTarget, new Path(targetTestRoot, "user/dirX")));
    fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"), FileContext.DEFAULT_PERM, false);
    Assert.assertTrue("New dir should be type dir", isDir(fcView, new Path("/user/dirX/dirY")));
    Assert.assertTrue("Target of new dir should be of type dir", isDir(fcTarget, new Path(targetTestRoot, "user/dirX/dirY")));
    // Delete the created dir
    Assert.assertTrue("Delete should succeed", fcView.delete(new Path("/user/dirX/dirY"), false));
    Assert.assertFalse("Deleted File should not exist", exists(fcView, new Path("/user/dirX/dirY")));
    Assert.assertFalse("Deleted Target should not exist", exists(fcTarget, new Path(targetTestRoot, "user/dirX/dirY")));
    Assert.assertTrue("Delete should succeed", fcView.delete(new Path("/user/dirX"), false));
    Assert.assertFalse("Deleted File should not exist", exists(fcView, new Path("/user/dirX")));
    Assert.assertFalse("Deleted Target should not exist", exists(fcTarget, new Path(targetTestRoot, "user/dirX")));
    // Rename a file 
    fileContextTestHelper.createFile(fcView, "/user/foo");
    fcView.rename(new Path("/user/foo"), new Path("/user/fooBar"));
    Assert.assertFalse("Renamed src should not exist", exists(fcView, new Path("/user/foo")));
    Assert.assertFalse(exists(fcTarget, new Path(targetTestRoot, "user/foo")));
    Assert.assertTrue(isFile(fcView, fileContextTestHelper.getTestRootPath(fcView, "/user/fooBar")));
    Assert.assertTrue(isFile(fcTarget, new Path(targetTestRoot, "user/fooBar")));
    fcView.mkdir(new Path("/user/dirFoo"), FileContext.DEFAULT_PERM, false);
    fcView.rename(new Path("/user/dirFoo"), new Path("/user/dirFooBar"));
    Assert.assertFalse("Renamed src should not exist", exists(fcView, new Path("/user/dirFoo")));
    Assert.assertFalse("Renamed src should not exist in target", exists(fcTarget, new Path(targetTestRoot, "user/dirFoo")));
    Assert.assertTrue("Renamed dest should  exist as dir", isDir(fcView, fileContextTestHelper.getTestRootPath(fcView, "/user/dirFooBar")));
    Assert.assertTrue("Renamed dest should  exist as dir in target", isDir(fcTarget, new Path(targetTestRoot, "user/dirFooBar")));
    // Make a directory under a directory that's mounted from the root of another FS
    fcView.mkdir(new Path("/targetRoot/dirFoo"), FileContext.DEFAULT_PERM, false);
    Assert.assertTrue(exists(fcView, new Path("/targetRoot/dirFoo")));
    boolean dirFooPresent = false;
    RemoteIterator<FileStatus> dirContents = fcView.listStatus(new Path("/targetRoot/"));
    while (dirContents.hasNext()) {
        FileStatus fileStatus = dirContents.next();
        if (fileStatus.getPath().getName().equals("dirFoo")) {
            dirFooPresent = true;
        }
    }
    Assert.assertTrue(dirFooPresent);
    RemoteIterator<LocatedFileStatus> dirLocatedContents = fcView.listLocatedStatus(new Path("/targetRoot/"));
    dirFooPresent = false;
    while (dirLocatedContents.hasNext()) {
        FileStatus fileStatus = dirLocatedContents.next();
        if (fileStatus.getPath().getName().equals("dirFoo")) {
            dirFooPresent = true;
        }
    }
    Assert.assertTrue(dirFooPresent);
}

Example 59 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestListFilesInFileContext method testSymbolicLinks.

/** Test when input patch has a symbolic links as its children */
@Test
public void testSymbolicLinks() throws IOException {
    writeFile(fc, FILE1, FILE_LEN);
    writeFile(fc, FILE2, FILE_LEN);
    writeFile(fc, FILE3, FILE_LEN);
    Path dir4 = new Path(TEST_DIR, "dir4");
    Path dir5 = new Path(dir4, "dir5");
    Path file4 = new Path(dir4, "file4");
    fc.createSymlink(DIR1, dir5, true);
    fc.createSymlink(FILE1, file4, true);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(dir4, true);
    LocatedFileStatus stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE2), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE3), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
    itor = fc.util().listFiles(dir4, false);
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
}

Example 60 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestListFilesInFileContext method testFile.

/** Test when input path is a file */
@Test
public void testFile() throws IOException {
    fc.mkdir(TEST_DIR, FsPermission.getDefault(), true);
    writeFile(fc, FILE1, FILE_LEN);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(FILE1, true);
    LocatedFileStatus stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
    itor = fc.util().listFiles(FILE1, false);
    stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
}