Examples with LocatedFileStatus org.apache.hadoop.fs.LocatedFileStatus used on opensource projects

Example 66 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project SpyGlass by ParallelAI.

the class JobLibLoader method loadJars.

public static void loadJars(String libPathStr, Configuration config) {
    try {
        Path libPath = new Path(libPathStr);
        FileSystem fs = FileSystem.get(config);
        RemoteIterator<LocatedFileStatus> itr = fs.listFiles(libPath, true);
        while (itr.hasNext()) {
            LocatedFileStatus f = itr.next();
            if (!f.isDirectory() && f.getPath().getName().endsWith("jar")) {
                logger.info("Loading Jar : " + f.getPath().getName());
                DistributedCache.addFileToClassPath(f.getPath(), config);
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        logger.error(e.toString());
    }
}

Example 67 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project presto by prestodb.

the class BackgroundHiveSplitLoader method loadSplits.

private CompletableFuture<?> loadSplits() throws IOException {
    HiveFileIterator files = fileIterators.poll();
    if (files == null) {
        HivePartitionMetadata partition = partitions.poll();
        if (partition == null) {
            return COMPLETED_FUTURE;
        }
        loadPartition(partition);
        return COMPLETED_FUTURE;
    }
    while (files.hasNext() && !stopped) {
        LocatedFileStatus file = files.next();
        if (isDirectory(file)) {
            if (recursiveDirWalkerEnabled) {
                HiveFileIterator fileIterator = new HiveFileIterator(file.getPath(), files.getFileSystem(), files.getDirectoryLister(), files.getNamenodeStats(), files.getPartitionName(), files.getInputFormat(), files.getSchema(), files.getPartitionKeys(), files.getEffectivePredicate(), files.getColumnCoercions());
                fileIterators.add(fileIterator);
            }
        } else {
            boolean splittable = isSplittable(files.getInputFormat(), hdfsEnvironment.getFileSystem(session.getUser(), file.getPath()), file.getPath());
            CompletableFuture<?> future = hiveSplitSource.addToQueue(createHiveSplitIterator(files.getPartitionName(), file.getPath().toString(), file.getBlockLocations(), 0, file.getLen(), files.getSchema(), files.getPartitionKeys(), splittable, session, OptionalInt.empty(), files.getEffectivePredicate(), files.getColumnCoercions()));
            if (!future.isDone()) {
                fileIterators.addFirst(files);
                return future;
            }
        }
    }
    // No need to put the iterator back, since it's either empty or we've stopped
    return COMPLETED_FUTURE;
}

Example 68 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project apex-core by apache.

the class RecordingsAgent method getRecordingInfo.

public List<RecordingInfo> getRecordingInfo(String appId) {
    List<RecordingInfo> result = new ArrayList<>();
    String dir = getRecordingsDirectory(appId);
    if (dir == null) {
        return result;
    }
    Path path = new Path(dir);
    try {
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        if (!fileStatus.isDirectory()) {
            return result;
        }
        RemoteIterator<LocatedFileStatus> ri = stramAgent.getFileSystem().listLocatedStatus(path);
        while (ri.hasNext()) {
            LocatedFileStatus lfs = ri.next();
            if (lfs.isDirectory()) {
                try {
                    String opId = lfs.getPath().getName();
                    result.addAll(getRecordingInfo(appId, opId));
                } catch (NumberFormatException ex) {
                // ignore
                }
            }
        }
    } catch (IOException ex) {
        LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
        return result;
    }
    return result;
}

Example 69 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project apex-core by apache.

the class FSAgent method listFilesInfo.

public List<LocatedFileStatus> listFilesInfo(String dir) throws IOException {
    List<LocatedFileStatus> files = new ArrayList<>();
    Path path = new Path(dir);
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    if (!fileStatus.isDirectory()) {
        throw new FileNotFoundException("Cannot read directory " + dir);
    }
    RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
    while (it.hasNext()) {
        LocatedFileStatus lfs = it.next();
        files.add(lfs);
    }
    return files;
}

Example 70 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project drill by apache.

the class TestCTTAS method checkPermission.

private void checkPermission(String tmpTableName) throws IOException {
    File[] files = findTemporaryTableLocation(tmpTableName);
    assertEquals("Only one directory should match temporary table name " + tmpTableName, 1, files.length);
    Path tmpTablePath = new Path(files[0].toURI().getPath());
    assertEquals("Directory permission should match", expectedFolderPermission, fs.getFileStatus(tmpTablePath).getPermission());
    RemoteIterator<LocatedFileStatus> fileIterator = fs.listFiles(tmpTablePath, false);
    while (fileIterator.hasNext()) {
        assertEquals("File permission should match", expectedFilePermission, fileIterator.next().getPermission());
    }
}