Examples with Readable com.baidu.hugegraph.loader.reader.Readable used on opensource projects

Example 1 with Readable

use of com.baidu.hugegraph.loader.reader.Readable in project incubator-hugegraph-toolchain by apache.

the class FileReader method init.

@Override
public void init(LoadContext context, InputStruct struct) throws InitException {
    this.progress(context, struct);
    List<Readable> readableList;
    try {
        readableList = this.scanReadables();
        // Sort readable files by name
        readableList.sort(Comparator.comparing(Readable::name));
    } catch (IOException e) {
        throw new InitException("Failed to scan readable files for '%s'", e, this.source);
    }
    this.readables = readableList.iterator();
    this.fetcher = this.createLineFetcher();
    this.fetcher.readHeaderIfNeeded(readableList);
}

Example 2 with Readable

use of com.baidu.hugegraph.loader.reader.Readable in project incubator-hugegraph-toolchain by apache.

the class OrcFileLineFetcher method readHeader.

@Override
public String[] readHeader(List<Readable> readables) {
    Readable readable = readables.get(0);
    this.openReader(readable);
    StructObjectInspector inspector;
    try {
        inspector = (StructObjectInspector) this.reader.getObjectInspector();
        return this.parseHeader(inspector);
    } finally {
        try {
            this.closeReader();
        } catch (IOException e) {
            LOG.warn("Failed to close reader of '{}'", readable);
        }
    }
}

Example 3 with Readable

use of com.baidu.hugegraph.loader.reader.Readable in project incubator-hugegraph-toolchain by apache.

the class HDFSFileReader method scanReadables.

@Override
protected List<Readable> scanReadables() throws IOException {
    Path path = new Path(this.source().path());
    FileFilter filter = this.source().filter();
    List<Readable> paths = new ArrayList<>();
    if (this.hdfs.isFile(path)) {
        if (!filter.reserved(path.getName())) {
            throw new LoadException("Please check path name and extensions, ensure " + "that at least one path is available for reading");
        }
        paths.add(new HDFSFile(this.hdfs, path));
    } else {
        assert this.hdfs.isDirectory(path);
        FileStatus[] statuses = this.hdfs.listStatus(path);
        Path[] subPaths = FileUtil.stat2Paths(statuses);
        for (Path subPath : subPaths) {
            if (filter.reserved(subPath.getName())) {
                paths.add(new HDFSFile(this.hdfs, subPath));
            }
        }
    }
    return paths;
}

Example 4 with Readable

use of com.baidu.hugegraph.loader.reader.Readable in project incubator-hugegraph-toolchain by apache.

the class LocalFileReader method scanReadables.

@Override
protected List<Readable> scanReadables() {
    File file = FileUtils.getFile(this.source().path());
    checkExistAndReadable(file);
    FileFilter filter = this.source().filter();
    List<Readable> files = new ArrayList<>();
    if (file.isFile()) {
        if (!filter.reserved(file.getName())) {
            throw new LoadException("Please check file name and extensions, ensure " + "that at least one file is available for reading");
        }
        files.add(new LocalFile(file));
    } else {
        assert file.isDirectory();
        File[] subFiles = file.listFiles();
        if (subFiles == null) {
            throw new LoadException("Error while listing the files of " + "path '%s'", file);
        }
        for (File subFile : subFiles) {
            if (filter.reserved(subFile.getName())) {
                files.add(new LocalFile(subFile));
            }
        }
    }
    return files;
}

Example 5 with Readable

use of com.baidu.hugegraph.loader.reader.Readable in project incubator-hugegraph-toolchain by apache.

the class ParquetFileLineFetcher method readHeader.

@Override
public String[] readHeader(List<Readable> readables) {
    Readable readable = readables.get(0);
    this.openReader(readables.get(0));
    try {
        return this.parseHeader(this.schema);
    } finally {
        try {
            this.closeReader();
        } catch (IOException e) {
            LOG.warn("Failed to close reader of '{}'", readable);
        }
    }
}