Examples with LocatedFileStatus org.apache.hadoop.fs.LocatedFileStatus used on opensource projects

Example 96 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class FileInputFormat method singleThreadedListStatus.

private List<FileStatus> singleThreadedListStatus(JobContext job, Path[] dirs, PathFilter inputFilter, boolean recursive) throws IOException {
    List<FileStatus> result = new ArrayList<FileStatus>();
    List<IOException> errors = new ArrayList<IOException>();
    for (int i = 0; i < dirs.length; ++i) {
        Path p = dirs[i];
        FileSystem fs = p.getFileSystem(job.getConfiguration());
        FileStatus[] matches = fs.globStatus(p, inputFilter);
        if (matches == null) {
            errors.add(new IOException("Input path does not exist: " + p));
        } else if (matches.length == 0) {
            errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
        } else {
            for (FileStatus globStat : matches) {
                if (globStat.isDirectory()) {
                    RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(globStat.getPath());
                    while (iter.hasNext()) {
                        LocatedFileStatus stat = iter.next();
                        if (inputFilter.accept(stat.getPath())) {
                            if (recursive && stat.isDirectory()) {
                                addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
                            } else {
                                result.add(stat);
                            }
                        }
                    }
                } else {
                    result.add(globStat);
                }
            }
        }
    }
    if (!errors.isEmpty()) {
        throw new InvalidInputException(errors);
    }
    return result;
}

Example 97 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project drill by apache.

the class TestCTTAS method checkPermission.

private void checkPermission(String tmpTableName) throws IOException {
    File[] files = findTemporaryTableLocation(tmpTableName);
    assertEquals("Only one directory should match temporary table name " + tmpTableName, 1, files.length);
    Path tmpTablePath = new Path(files[0].toURI().getPath());
    assertEquals("Directory permission should match", expectedFolderPermission, fs.getFileStatus(tmpTablePath).getPermission());
    RemoteIterator<LocatedFileStatus> fileIterator = fs.listFiles(tmpTablePath, false);
    while (fileIterator.hasNext()) {
        assertEquals("File permission should match", expectedFilePermission, fileIterator.next().getPermission());
    }
}

Example 98 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project metron by apache.

the class PcapJob method readResults.

/**
 * Returns a lazily-read Iterable over a set of sequence files
 */
private SequenceFileIterable readResults(Path outputPath, Configuration config, FileSystem fs) throws IOException {
    List<Path> files = new ArrayList<>();
    for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(outputPath, false); it.hasNext(); ) {
        Path p = it.next().getPath();
        if (p.getName().equals("_SUCCESS")) {
            fs.delete(p, false);
            continue;
        }
        files.add(p);
    }
    LOG.debug("Output path={}", outputPath);
    Collections.sort(files, (o1, o2) -> o1.getName().compareTo(o2.getName()));
    return new SequenceFileIterable(files, config);
}

Example 99 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestListFilesInFileContext method testSymbolicLinks.

/** Test when input patch has a symbolic links as its children */
@Test
public void testSymbolicLinks() throws IOException {
    writeFile(fc, FILE1, FILE_LEN);
    writeFile(fc, FILE2, FILE_LEN);
    writeFile(fc, FILE3, FILE_LEN);
    Path dir4 = new Path(TEST_DIR, "dir4");
    Path dir5 = new Path(dir4, "dir5");
    Path file4 = new Path(dir4, "file4");
    fc.createSymlink(DIR1, dir5, true);
    fc.createSymlink(FILE1, file4, true);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(dir4, true);
    LocatedFileStatus stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE2), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE3), stat.getPath());
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
    itor = fc.util().listFiles(dir4, false);
    stat = itor.next();
    assertTrue(stat.isFile());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertFalse(itor.hasNext());
}

Example 100 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestListFilesInFileContext method testFile.

/** Test when input path is a file */
@Test
public void testFile() throws IOException {
    fc.mkdir(TEST_DIR, FsPermission.getDefault(), true);
    writeFile(fc, FILE1, FILE_LEN);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(FILE1, true);
    LocatedFileStatus stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
    itor = fc.util().listFiles(FILE1, false);
    stat = itor.next();
    assertFalse(itor.hasNext());
    assertTrue(stat.isFile());
    assertEquals(FILE_LEN, stat.getLen());
    assertEquals(fc.makeQualified(FILE1), stat.getPath());
    assertEquals(1, stat.getBlockLocations().length);
}