Examples with ExtendedFileSystem com.facebook.presto.hive.filesystem.ExtendedFileSystem used on opensource projects

Example 11 with ExtendedFileSystem

use of com.facebook.presto.hive.filesystem.ExtendedFileSystem in project presto by prestodb.

the class HiveCachingHdfsConfiguration method getConfiguration.

@Override
public Configuration getConfiguration(HdfsContext context, URI uri) {
    @SuppressWarnings("resource") Configuration config = new CachingJobConf((factoryConfig, factoryUri) -> {
        try {
            FileSystem fileSystem = (new Path(factoryUri)).getFileSystem(hiveHdfsConfiguration.getConfiguration(context, factoryUri));
            checkState(fileSystem instanceof ExtendedFileSystem);
            return cacheFactory.createCachingFileSystem(factoryConfig, factoryUri, (ExtendedFileSystem) fileSystem, cacheManager, context.getSession().map(HiveSessionProperties::isCacheEnabled).orElse(cacheConfig.isCachingEnabled()), cacheConfig.getCacheType(), cacheConfig.isValidationEnabled());
        } catch (IOException e) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, "cannot create caching file system", e);
        }
    });
    Configuration defaultConfig = hiveHdfsConfiguration.getConfiguration(context, uri);
    copy(defaultConfig, config);
    return config;
}

Example 12 with ExtendedFileSystem

use of com.facebook.presto.hive.filesystem.ExtendedFileSystem in project presto by prestodb.

the class ManifestPartitionLoader method validateManifest.

private void validateManifest(HivePartitionMetadata partition, Path path, List<String> manifestFileNames, List<Long> manifestFileSizes) throws IOException {
    ExtendedFileSystem fileSystem = hdfsEnvironment.getFileSystem(hdfsContext, path);
    HiveDirectoryContext hiveDirectoryContext = new HiveDirectoryContext(recursiveDirWalkerEnabled ? RECURSE : IGNORED, false);
    Iterator<HiveFileInfo> fileInfoIterator = directoryLister.list(fileSystem, table, path, namenodeStats, ignore -> true, hiveDirectoryContext);
    int fileCount = 0;
    while (fileInfoIterator.hasNext()) {
        HiveFileInfo fileInfo = fileInfoIterator.next();
        String fileName = fileInfo.getPath().getName();
        if (!manifestFileNames.contains(fileName)) {
            throw new PrestoException(MALFORMED_HIVE_FILE_STATISTICS, format("Filename = %s not stored in manifest. Partition = %s, TableName = %s", fileName, partition.getHivePartition().getPartitionId(), table.getTableName()));
        }
        int index = manifestFileNames.indexOf(fileName);
        if (!manifestFileSizes.get(index).equals(fileInfo.getLength())) {
            throw new PrestoException(MALFORMED_HIVE_FILE_STATISTICS, format("FilesizeFromManifest = %s is not equal to FilesizeFromStorage = %s. File = %s, Partition = %s, TableName = %s", manifestFileSizes.get(index), fileInfo.getLength(), fileName, partition.getHivePartition().getPartitionId(), table.getTableName()));
        }
        fileCount++;
    }
    if (fileCount != manifestFileNames.size()) {
        throw new PrestoException(MALFORMED_HIVE_FILE_STATISTICS, format("Number of files in Manifest = %s is not equal to Number of files in storage = %s. Partition = %s, TableName = %s", manifestFileNames.size(), fileCount, partition.getHivePartition().getPartitionId(), table.getTableName()));
    }
}