Examples with FileFilter com.baidu.hugegraph.loader.source.file.FileFilter used on opensource projects

Example 1 with FileFilter

use of com.baidu.hugegraph.loader.source.file.FileFilter in project hugegraph-computer by hugegraph.

the class LoaderFileInputSplitFetcher method scanHdfsPaths.

private List<String> scanHdfsPaths(HDFSSource hdfsSource) {
    List<String> paths = new ArrayList<>();
    try {
        Configuration configuration = this.loadConfiguration(hdfsSource);
        this.enableKerberos(hdfsSource, configuration);
        try (FileSystem hdfs = FileSystem.get(configuration)) {
            Path path = new Path(hdfsSource.path());
            FileFilter filter = hdfsSource.filter();
            if (hdfs.getFileStatus(path).isFile()) {
                if (!filter.reserved(path.getName())) {
                    throw new ComputerException("Please check path name and extensions, ensure " + "that at least one path is available for reading");
                }
                paths.add(path.toString());
            } else {
                assert hdfs.getFileStatus(path).isDirectory();
                FileStatus[] statuses = hdfs.listStatus(path);
                Path[] subPaths = FileUtil.stat2Paths(statuses);
                for (Path subPath : subPaths) {
                    if (filter.reserved(subPath.getName())) {
                        paths.add(subPath.toString());
                    }
                }
            }
        }
    } catch (Throwable throwable) {
        throw new ComputerException("Failed to scanPaths", throwable);
    }
    return paths;
}

Example 2 with FileFilter

use of com.baidu.hugegraph.loader.source.file.FileFilter in project hugegraph-computer by hugegraph.

the class LoaderFileInputSplitFetcher method scanLocalPaths.

private List<String> scanLocalPaths(FileSource source) {
    List<String> paths = new ArrayList<>();
    File file = FileUtils.getFile(source.path());
    FileFilter filter = source.filter();
    if (file.isFile()) {
        if (!filter.reserved(file.getName())) {
            throw new LoadException("Please check file name and extensions, ensure " + "that at least one file is available for reading");
        }
        paths.add(file.getAbsolutePath());
    } else {
        assert file.isDirectory();
        File[] subFiles = file.listFiles();
        if (subFiles == null) {
            throw new LoadException("Error while listing the files of " + "path '%s'", file);
        }
        for (File subFile : subFiles) {
            if (filter.reserved(subFile.getName())) {
                paths.add(subFile.getAbsolutePath());
            }
        }
    }
    return paths;
}

Example 3 with FileFilter

use of com.baidu.hugegraph.loader.source.file.FileFilter in project incubator-hugegraph-toolchain by apache.

the class HDFSFileReader method scanReadables.

@Override
protected List<Readable> scanReadables() throws IOException {
    Path path = new Path(this.source().path());
    FileFilter filter = this.source().filter();
    List<Readable> paths = new ArrayList<>();
    if (this.hdfs.isFile(path)) {
        if (!filter.reserved(path.getName())) {
            throw new LoadException("Please check path name and extensions, ensure " + "that at least one path is available for reading");
        }
        paths.add(new HDFSFile(this.hdfs, path));
    } else {
        assert this.hdfs.isDirectory(path);
        FileStatus[] statuses = this.hdfs.listStatus(path);
        Path[] subPaths = FileUtil.stat2Paths(statuses);
        for (Path subPath : subPaths) {
            if (filter.reserved(subPath.getName())) {
                paths.add(new HDFSFile(this.hdfs, subPath));
            }
        }
    }
    return paths;
}

Example 4 with FileFilter

use of com.baidu.hugegraph.loader.source.file.FileFilter in project incubator-hugegraph-toolchain by apache.

the class LocalFileReader method scanReadables.

@Override
protected List<Readable> scanReadables() {
    File file = FileUtils.getFile(this.source().path());
    checkExistAndReadable(file);
    FileFilter filter = this.source().filter();
    List<Readable> files = new ArrayList<>();
    if (file.isFile()) {
        if (!filter.reserved(file.getName())) {
            throw new LoadException("Please check file name and extensions, ensure " + "that at least one file is available for reading");
        }
        files.add(new LocalFile(file));
    } else {
        assert file.isDirectory();
        File[] subFiles = file.listFiles();
        if (subFiles == null) {
            throw new LoadException("Error while listing the files of " + "path '%s'", file);
        }
        for (File subFile : subFiles) {
            if (filter.reserved(subFile.getName())) {
                files.add(new LocalFile(subFile));
            }
        }
    }
    return files;
}