Search in sources :

Example 1 with HIVE_INVALID_BUCKET_FILES

use of com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES in project presto by prestodb.

the class StoragePartitionLoader method getBucketedSplits.

private List<InternalHiveSplit> getBucketedSplits(Path path, ExtendedFileSystem fileSystem, InternalHiveSplitFactory splitFactory, BucketSplitInfo bucketSplitInfo, Optional<HiveSplit.BucketConversion> bucketConversion, String partitionName, boolean splittable, PathFilter pathFilter) {
    int readBucketCount = bucketSplitInfo.getReadBucketCount();
    int tableBucketCount = bucketSplitInfo.getTableBucketCount();
    int partitionBucketCount = bucketConversion.map(HiveSplit.BucketConversion::getPartitionBucketCount).orElse(tableBucketCount);
    int bucketCount = max(readBucketCount, partitionBucketCount);
    checkState(readBucketCount <= tableBucketCount, "readBucketCount(%s) should be less than or equal to tableBucketCount(%s)", readBucketCount, tableBucketCount);
    // list all files in the partition
    List<HiveFileInfo> fileInfos = new ArrayList<>(partitionBucketCount);
    try {
        Iterators.addAll(fileInfos, directoryLister.list(fileSystem, table, path, namenodeStats, pathFilter, new HiveDirectoryContext(FAIL, isUseListDirectoryCache(session))));
    } catch (HiveFileIterator.NestedDirectoryNotAllowedException e) {
        // Fail here to be on the safe side. This seems to be the same as what Hive does
        throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. Found sub-directory in bucket directory for partition: %s", table.getSchemaTableName(), partitionName));
    }
    ListMultimap<Integer, HiveFileInfo> bucketToFileInfo = ArrayListMultimap.create();
    if (!shouldCreateFilesForMissingBuckets(table, session)) {
        fileInfos.stream().forEach(fileInfo -> {
            String fileName = fileInfo.getPath().getName();
            OptionalInt bucket = getBucketNumber(fileName);
            if (bucket.isPresent()) {
                bucketToFileInfo.put(bucket.getAsInt(), fileInfo);
            } else {
                throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("invalid hive bucket file name: %s", fileName));
            }
        });
    } else {
        // build mapping of file name to bucket
        for (HiveFileInfo file : fileInfos) {
            String fileName = file.getPath().getName();
            OptionalInt bucket = getBucketNumber(fileName);
            if (bucket.isPresent()) {
                bucketToFileInfo.put(bucket.getAsInt(), file);
                continue;
            }
            // legacy mode requires exactly one file per bucket
            if (fileInfos.size() != partitionBucketCount) {
                throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. File '%s' does not match the standard naming pattern, and the number " + "of files in the directory (%s) does not match the declared bucket count (%s) for partition: %s", table.getSchemaTableName(), fileName, fileInfos.size(), partitionBucketCount, partitionName));
            }
            if (fileInfos.get(0).getPath().getName().matches("\\d+")) {
                try {
                    // File names are integer if they are created when file_renaming_enabled is set to true
                    fileInfos.sort(Comparator.comparingInt(fileInfo -> Integer.parseInt(fileInfo.getPath().getName())));
                } catch (NumberFormatException e) {
                    throw new PrestoException(HIVE_INVALID_FILE_NAMES, format("Hive table '%s' is corrupt. Some of the filenames in the partition: %s are not integers", new SchemaTableName(table.getDatabaseName(), table.getTableName()), partitionName));
                }
            } else {
                // Sort FileStatus objects (instead of, e.g., fileStatus.getPath().toString). This matches org.apache.hadoop.hive.ql.metadata.Table.getSortedPaths
                fileInfos.sort(null);
            }
            // Use position in sorted list as the bucket number
            bucketToFileInfo.clear();
            for (int i = 0; i < fileInfos.size(); i++) {
                bucketToFileInfo.put(i, fileInfos.get(i));
            }
            break;
        }
    }
    // convert files internal splits
    List<InternalHiveSplit> splitList = new ArrayList<>();
    for (int bucketNumber = 0; bucketNumber < bucketCount; bucketNumber++) {
        // Physical bucket #. This determine file name. It also determines the order of splits in the result.
        int partitionBucketNumber = bucketNumber % partitionBucketCount;
        if (!bucketToFileInfo.containsKey(partitionBucketNumber)) {
            continue;
        }
        // Logical bucket #. Each logical bucket corresponds to a "bucket" from engine's perspective.
        int readBucketNumber = bucketNumber % readBucketCount;
        boolean containsIneligibleTableBucket = false;
        List<Integer> eligibleTableBucketNumbers = new ArrayList<>();
        for (int tableBucketNumber = bucketNumber % tableBucketCount; tableBucketNumber < tableBucketCount; tableBucketNumber += bucketCount) {
            // table bucket number: this is used for evaluating "$bucket" filters.
            if (bucketSplitInfo.isTableBucketEnabled(tableBucketNumber)) {
                eligibleTableBucketNumbers.add(tableBucketNumber);
            } else {
                containsIneligibleTableBucket = true;
            }
        }
        if (!eligibleTableBucketNumbers.isEmpty() && containsIneligibleTableBucket) {
            throw new PrestoException(NOT_SUPPORTED, "The bucket filter cannot be satisfied. There are restrictions on the bucket filter when all the following is true: " + "1. a table has a different buckets count as at least one of its partitions that is read in this query; " + "2. the table has a different but compatible bucket number with another table in the query; " + "3. some buckets of the table is filtered out from the query, most likely using a filter on \"$bucket\". " + "(table name: " + table.getTableName() + ", table bucket count: " + tableBucketCount + ", " + "partition bucket count: " + partitionBucketCount + ", effective reading bucket count: " + readBucketCount + ")");
        }
        if (!eligibleTableBucketNumbers.isEmpty()) {
            for (HiveFileInfo fileInfo : bucketToFileInfo.get(partitionBucketNumber)) {
                eligibleTableBucketNumbers.stream().map(tableBucketNumber -> splitFactory.createInternalHiveSplit(fileInfo, readBucketNumber, tableBucketNumber, splittable)).forEach(optionalSplit -> optionalSplit.ifPresent(splitList::add));
            }
        }
    }
    return splitList;
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) LoadingCache(com.google.common.cache.LoadingCache) ListMultimap(com.google.common.collect.ListMultimap) HiveSessionProperties.isFileSplittable(com.facebook.presto.hive.HiveSessionProperties.isFileSplittable) HiveSessionProperties.isUseListDirectoryCache(com.facebook.presto.hive.HiveSessionProperties.isUseListDirectoryCache) FileStatus(org.apache.hadoop.fs.FileStatus) IntPredicate(java.util.function.IntPredicate) HiveUtil.getHeaderCount(com.facebook.presto.hive.HiveUtil.getHeaderCount) HiveUtil.getInputFormat(com.facebook.presto.hive.HiveUtil.getInputFormat) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) FileSplit(org.apache.hadoop.mapred.FileSplit) CharStreams(com.google.common.io.CharStreams) Configuration(org.apache.hadoop.conf.Configuration) InputFormat(org.apache.hadoop.mapred.InputFormat) Path(org.apache.hadoop.fs.Path) HiveMetadata.shouldCreateFilesForMissingBuckets(com.facebook.presto.hive.HiveMetadata.shouldCreateFilesForMissingBuckets) HIVE_INVALID_FILE_NAMES(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_FILE_NAMES) Function(com.google.common.base.Function) FileInputFormat(org.apache.hadoop.mapred.FileInputFormat) ExtendedFileSystem(com.facebook.presto.hive.filesystem.ExtendedFileSystem) HiveWriterFactory.getBucketNumber(com.facebook.presto.hive.HiveWriterFactory.getBucketNumber) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) HiveSessionProperties.isStreamingAggregationEnabled(com.facebook.presto.hive.HiveSessionProperties.isStreamingAggregationEnabled) S3SelectPushdown.shouldEnablePushdownForTable(com.facebook.presto.hive.S3SelectPushdown.shouldEnablePushdownForTable) StandardCharsets(java.nio.charset.StandardCharsets) String.format(java.lang.String.format) HiveSessionProperties.getMaxInitialSplitSize(com.facebook.presto.hive.HiveSessionProperties.getMaxInitialSplitSize) FAIL(com.facebook.presto.hive.NestedDirectoryPolicy.FAIL) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ConnectorSession(com.facebook.presto.spi.ConnectorSession) CacheLoader(com.google.common.cache.CacheLoader) List(java.util.List) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) HIDDEN_FILES_PATH_FILTER(org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER) Optional(java.util.Optional) Math.max(java.lang.Math.max) CacheBuilder(com.google.common.cache.CacheBuilder) TextInputFormat(org.apache.hadoop.mapred.TextInputFormat) Table(com.facebook.presto.hive.metastore.Table) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) HiveUtil.getFooterCount(com.facebook.presto.hive.HiveUtil.getFooterCount) PathFilter(org.apache.hadoop.fs.PathFilter) MetastoreUtil.getPartitionLocation(com.facebook.presto.hive.metastore.MetastoreUtil.getPartitionLocation) PrestoException(com.facebook.presto.spi.PrestoException) Deque(java.util.Deque) HiveSessionProperties.getNodeSelectionStrategy(com.facebook.presto.hive.HiveSessionProperties.getNodeSelectionStrategy) OptionalInt(java.util.OptionalInt) Iterators(com.google.common.collect.Iterators) Partition(com.facebook.presto.hive.metastore.Partition) SymlinkTextInputFormat(org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat) ArrayList(java.util.ArrayList) IGNORED(com.facebook.presto.hive.NestedDirectoryPolicy.IGNORED) ImmutableList(com.google.common.collect.ImmutableList) HiveUtil.shouldUseFileSplitsFromInputFormat(com.facebook.presto.hive.HiveUtil.shouldUseFileSplitsFromInputFormat) ConfigurationUtils.toJobConf(com.facebook.presto.hive.util.ConfigurationUtils.toJobConf) Objects.requireNonNull(java.util.Objects.requireNonNull) HIVE_BAD_DATA(com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA) RECURSE(com.facebook.presto.hive.NestedDirectoryPolicy.RECURSE) HIVE_INVALID_BUCKET_FILES(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) Storage(com.facebook.presto.hive.metastore.Storage) Properties(java.util.Properties) Iterator(java.util.Iterator) InternalHiveSplitFactory(com.facebook.presto.hive.util.InternalHiveSplitFactory) HiveColumnHandle.pathColumnHandle(com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle) HoodieROTablePathFilter(org.apache.hudi.hadoop.HoodieROTablePathFilter) IOException(java.io.IOException) HiveFileIterator(com.facebook.presto.hive.util.HiveFileIterator) InputStreamReader(java.io.InputStreamReader) Domain(com.facebook.presto.common.predicate.Domain) JobConf(org.apache.hadoop.mapred.JobConf) Streams.stream(com.google.common.collect.Streams.stream) InputSplit(org.apache.hadoop.mapred.InputSplit) BufferedReader(java.io.BufferedReader) Maps.fromProperties(com.google.common.collect.Maps.fromProperties) Comparator(java.util.Comparator) HiveBucketing.getVirtualBucketNumber(com.facebook.presto.hive.HiveBucketing.getVirtualBucketNumber) HiveUtil.isHudiParquetInputFormat(com.facebook.presto.hive.HiveUtil.isHudiParquetInputFormat) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) ArrayList(java.util.ArrayList) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HiveFileIterator(com.facebook.presto.hive.util.HiveFileIterator)

Aggregations

Domain (com.facebook.presto.common.predicate.Domain)1 HiveBucketing.getVirtualBucketNumber (com.facebook.presto.hive.HiveBucketing.getVirtualBucketNumber)1 HiveColumnHandle.pathColumnHandle (com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle)1 HIVE_BAD_DATA (com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA)1 HIVE_INVALID_BUCKET_FILES (com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES)1 HIVE_INVALID_FILE_NAMES (com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_FILE_NAMES)1 HiveMetadata.shouldCreateFilesForMissingBuckets (com.facebook.presto.hive.HiveMetadata.shouldCreateFilesForMissingBuckets)1 HiveSessionProperties.getMaxInitialSplitSize (com.facebook.presto.hive.HiveSessionProperties.getMaxInitialSplitSize)1 HiveSessionProperties.getNodeSelectionStrategy (com.facebook.presto.hive.HiveSessionProperties.getNodeSelectionStrategy)1 HiveSessionProperties.isFileSplittable (com.facebook.presto.hive.HiveSessionProperties.isFileSplittable)1 HiveSessionProperties.isStreamingAggregationEnabled (com.facebook.presto.hive.HiveSessionProperties.isStreamingAggregationEnabled)1 HiveSessionProperties.isUseListDirectoryCache (com.facebook.presto.hive.HiveSessionProperties.isUseListDirectoryCache)1 HiveUtil.getFooterCount (com.facebook.presto.hive.HiveUtil.getFooterCount)1 HiveUtil.getHeaderCount (com.facebook.presto.hive.HiveUtil.getHeaderCount)1 HiveUtil.getInputFormat (com.facebook.presto.hive.HiveUtil.getInputFormat)1 HiveUtil.isHudiParquetInputFormat (com.facebook.presto.hive.HiveUtil.isHudiParquetInputFormat)1 HiveUtil.shouldUseFileSplitsFromInputFormat (com.facebook.presto.hive.HiveUtil.shouldUseFileSplitsFromInputFormat)1 HiveWriterFactory.getBucketNumber (com.facebook.presto.hive.HiveWriterFactory.getBucketNumber)1 FAIL (com.facebook.presto.hive.NestedDirectoryPolicy.FAIL)1 IGNORED (com.facebook.presto.hive.NestedDirectoryPolicy.IGNORED)1