Examples with HdfsFileStatusWithId org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId used on opensource projects

Example 21 with HdfsFileStatusWithId

use of org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId in project hive by apache.

the class AcidUtils method getAcidFilesForStats.

public static List<FileStatus> getAcidFilesForStats(Table table, Path dir, Configuration jc, FileSystem fs) throws IOException {
    List<FileStatus> fileList = new ArrayList<>();
    ValidWriteIdList idList = AcidUtils.getTableValidWriteIdList(jc, AcidUtils.getFullTableName(table.getDbName(), table.getTableName()));
    if (idList == null) {
        LOG.warn("Cannot get ACID state for " + table.getDbName() + "." + table.getTableName() + " from " + jc.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY));
        return null;
    }
    if (fs == null) {
        fs = dir.getFileSystem(jc);
    }
    // Collect the all of the files/dirs
    Map<Path, HdfsDirSnapshot> hdfsDirSnapshots = AcidUtils.getHdfsDirSnapshots(fs, dir);
    AcidDirectory acidInfo = AcidUtils.getAcidState(fs, dir, jc, idList, null, false, hdfsDirSnapshots);
    // Assume that for an MM table, or if there's only the base directory, we are good.
    if (!acidInfo.getCurrentDirectories().isEmpty() && AcidUtils.isFullAcidTable(table)) {
        Utilities.FILE_OP_LOGGER.warn("Computing stats for an ACID table; stats may be inaccurate");
    }
    for (HdfsFileStatusWithId hfs : acidInfo.getOriginalFiles()) {
        fileList.add(hfs.getFileStatus());
    }
    for (ParsedDelta delta : acidInfo.getCurrentDirectories()) {
        fileList.addAll(hdfsDirSnapshots.get(delta.getPath()).getFiles());
    }
    if (acidInfo.getBaseDirectory() != null) {
        fileList.addAll(hdfsDirSnapshots.get(acidInfo.getBaseDirectory()).getFiles());
    }
    return fileList;
}

Example 22 with HdfsFileStatusWithId

use of org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId in project hive by apache.

the class ExternalCache method generateTestFileId.

private Long generateTestFileId(final FileStatus fs, List<HdfsFileStatusWithId> files, int i) {
    final Long fileId = HdfsUtils.createTestFileId(fs.getPath().toUri().getPath(), fs, false, null);
    files.set(i, new HdfsFileStatusWithId() {

        @Override
        public FileStatus getFileStatus() {
            return fs;
        }

        @Override
        public Long getFileId() {
            return fileId;
        }
    });
    return fileId;
}