Examples with FileBasedHelperException org.apache.gobblin.source.extractor.filebased.FileBasedHelperException used on opensource projects

Example 1 with FileBasedHelperException

use of org.apache.gobblin.source.extractor.filebased.FileBasedHelperException in project incubator-gobblin by apache.

the class SftpFsHelper method getFileSize.

@Override
public long getFileSize(String filePath) throws FileBasedHelperException {
    try {
        ChannelSftp channelSftp = getSftpChannel();
        long fileSize = channelSftp.lstat(filePath).getSize();
        channelSftp.disconnect();
        return fileSize;
    } catch (SftpException e) {
        throw new FileBasedHelperException(String.format("Failed to get size for file at path %s due to error %s", filePath, e.getMessage()), e);
    }
}

Example 2 with FileBasedHelperException

use of org.apache.gobblin.source.extractor.filebased.FileBasedHelperException in project incubator-gobblin by apache.

the class SftpFsHelper method ls.

@Override
public List<String> ls(String path) throws FileBasedHelperException {
    try {
        List<String> list = new ArrayList<>();
        ChannelSftp channel = getSftpChannel();
        Vector<LsEntry> vector = channel.ls(path);
        for (LsEntry entry : vector) {
            list.add(entry.getFilename());
        }
        channel.disconnect();
        return list;
    } catch (SftpException e) {
        throw new FileBasedHelperException("Cannot execute ls command on sftp connection", e);
    }
}

Example 3 with FileBasedHelperException

use of org.apache.gobblin.source.extractor.filebased.FileBasedHelperException in project incubator-gobblin by apache.

the class SftpLightWeightFileSystem method listStatus.

@Override
public FileStatus[] listStatus(Path path) throws IOException {
    try {
        List<String> fileNames = this.fsHelper.ls(HadoopUtils.toUriPath(path));
        List<FileStatus> status = Lists.newArrayListWithCapacity(fileNames.size());
        for (String name : fileNames) {
            Path filePath = new Path(name);
            if (VALID_PATH_FILTER.accept(filePath)) {
                status.add(getFileStatus(new Path(path, filePath)));
            }
        }
        return status.toArray(new FileStatus[status.size()]);
    } catch (FileBasedHelperException e) {
        throw new IOException(e);
    }
}

Example 4 with FileBasedHelperException

use of org.apache.gobblin.source.extractor.filebased.FileBasedHelperException in project incubator-gobblin by apache.

the class GoogleDriveFsHelper method getFileSize.

@Override
public long getFileSize(String fileId) throws FileBasedHelperException {
    Preconditions.checkNotNull(fileId, "fileId is required");
    Path p = new Path(fileId);
    try {
        FileStatus status = fileSystem.getFileStatus(p);
        return status.getLen();
    } catch (IOException e) {
        throw new FileBasedHelperException("Failed to get metadata on " + fileId, e);
    }
}

Example 5 with FileBasedHelperException

use of org.apache.gobblin.source.extractor.filebased.FileBasedHelperException in project incubator-gobblin by apache.

the class GoogleDriveFsHelper method ls.

/**
 * List files under folder ID recursively. Folder won't be included in the result. If there's no files under folder ID, it returns empty list.
 * If folder ID is not defined, it will provide files under root directory.
 * {@inheritDoc}
 * @see org.apache.gobblin.source.extractor.filebased.FileBasedHelper#ls(java.lang.String)
 */
@Override
public List<String> ls(String folderId) throws FileBasedHelperException {
    List<String> result = new ArrayList<>();
    if (StringUtils.isEmpty(folderId)) {
        folderId = "/";
    }
    Path p = new Path(folderId);
    FileStatus[] statusList = null;
    try {
        statusList = fileSystem.listStatus(p);
    } catch (FileNotFoundException e) {
        return result;
    } catch (IOException e) {
        throw new FileBasedHelperException("Falied to list status on path " + p + ", folderID: " + folderId, e);
    }
    for (FileStatus status : statusList) {
        if (status.isDirectory()) {
            result.addAll(ls(GoogleDriveFileSystem.toFileId(status.getPath())));
        } else {
            result.add(GoogleDriveFileSystem.toFileId(status.getPath()));
        }
    }
    return result;
}