use of io.trino.plugin.hive.util.HiveBucketing.BucketingVersion in project trino by trinodb.
the class HiveBucketProperty method fromStorageDescriptor.
public static Optional<HiveBucketProperty> fromStorageDescriptor(Map<String, String> tableParameters, StorageDescriptor storageDescriptor, String tablePartitionName) {
boolean bucketColsSet = storageDescriptor.isSetBucketCols() && !storageDescriptor.getBucketCols().isEmpty();
boolean numBucketsSet = storageDescriptor.isSetNumBuckets() && storageDescriptor.getNumBuckets() > 0;
if (!numBucketsSet) {
// In Hive, a table is considered as not bucketed when its bucketCols is set but its numBucket is not set.
return Optional.empty();
}
if (!bucketColsSet) {
throw new TrinoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set: " + tablePartitionName);
}
List<SortingColumn> sortedBy = ImmutableList.of();
if (storageDescriptor.isSetSortCols()) {
sortedBy = storageDescriptor.getSortCols().stream().map(order -> SortingColumn.fromMetastoreApiOrder(order, tablePartitionName)).collect(toImmutableList());
}
BucketingVersion bucketingVersion = HiveBucketing.getBucketingVersion(tableParameters);
return Optional.of(new HiveBucketProperty(storageDescriptor.getBucketCols(), bucketingVersion, storageDescriptor.getNumBuckets(), sortedBy));
}
use of io.trino.plugin.hive.util.HiveBucketing.BucketingVersion in project trino by trinodb.
the class TestHiveBucketing method testHashingCompare.
@Test
public void testHashingCompare() {
assertBucketEquals("string", "Trino rocks", 1132136730, -399107423);
assertEquals(HiveBucketing.getBucketNumber(1132136730, 4), 2);
assertEquals(HiveBucketing.getBucketNumber(-399107423, 4), 1);
assertBucketEquals("boolean", null, 0, 0);
assertBucketEquals("boolean", true, 1, 1);
assertBucketEquals("boolean", false, 0, 0);
assertBucketEquals("tinyint", null, 0, 0);
assertBucketEquals("tinyint", (byte) 5, 5, 5);
assertBucketEquals("tinyint", Byte.MIN_VALUE, -128, -128);
assertBucketEquals("tinyint", Byte.MAX_VALUE, 127, 127);
assertBucketEquals("smallint", null, 0, 0);
assertBucketEquals("smallint", (short) 300, 300, 2107031704);
assertBucketEquals("smallint", Short.MIN_VALUE, -32768, 1342976838);
assertBucketEquals("smallint", Short.MAX_VALUE, 32767, -684075052);
assertBucketEquals("int", null, 0, 0);
assertBucketEquals("int", 300_000, 300000, -678663480);
assertBucketEquals("int", Integer.MIN_VALUE, -2147483648, 1194881028);
assertBucketEquals("int", Integer.MAX_VALUE, 2147483647, 1133859967);
assertBucketEquals("bigint", null, 0, 0);
assertBucketEquals("bigint", 300_000_000_000L, -647710651, -888935297);
assertBucketEquals("bigint", Long.MIN_VALUE, -2147483648, 1728983947);
assertBucketEquals("bigint", Long.MAX_VALUE, -2147483648, -536577852);
assertBucketEquals("float", null, 0, 0);
assertBucketEquals("float", 12.34F, 1095069860, -381747602);
assertBucketEquals("float", -Float.MAX_VALUE, -8388609, 470252243);
assertBucketEquals("float", Float.MIN_VALUE, 1, 1206721797);
assertBucketEquals("float", Float.POSITIVE_INFINITY, 2139095040, -292175804);
assertBucketEquals("float", Float.NEGATIVE_INFINITY, -8388608, -1433270801);
assertBucketEquals("float", Float.NaN, 2143289344, -480354314);
// also a NaN
assertBucketEquals("float", intBitsToFloat(0xffc00000), 2143289344, -480354314);
// also a NaN
assertBucketEquals("float", intBitsToFloat(0x7fc00000), 2143289344, -480354314);
// also a NaN
assertBucketEquals("float", intBitsToFloat(0x7fc01234), 2143289344, -480354314);
// also a NaN
assertBucketEquals("float", intBitsToFloat(0xffc01234), 2143289344, -480354314);
assertBucketEquals("double", null, 0, 0);
assertBucketEquals("double", 12.34, 986311098, -2070733568);
assertBucketEquals("double", -Double.MAX_VALUE, 1048576, 14392725);
assertBucketEquals("double", Double.MIN_VALUE, 1, -8838199);
assertBucketEquals("double", Double.POSITIVE_INFINITY, 2146435072, 1614292060);
assertBucketEquals("double", Double.NEGATIVE_INFINITY, -1048576, 141388605);
assertBucketEquals("double", Double.NaN, 2146959360, 1138026565);
// also a NaN
assertBucketEquals("double", longBitsToDouble(0xfff8000000000000L), 2146959360, 1138026565);
// also a NaN
assertBucketEquals("double", longBitsToDouble(0x7ff8123412341234L), 2146959360, 1138026565);
// also a NaN
assertBucketEquals("double", longBitsToDouble(0xfff8123412341234L), 2146959360, 1138026565);
assertBucketEquals("varchar(15)", null, 0, 0);
assertBucketEquals("varchar(15)", "", 1, -965378730);
assertBucketEquals("varchar(15)", "test string", -189841218, -138301454);
// 3-byte UTF-8 sequences (in Basic Plane, i.e. Plane 0)
assertBucketEquals("varchar(15)", "\u5f3a\u5927\u7684Trino\u5f15\u64ce", 1899852420, 1784416344);
// 4 code points: 20FFC - 20FFF. 4-byte UTF-8 sequences in Supplementary Plane 2
assertBucketEquals("varchar(15)", "\uD843\uDFFC\uD843\uDFFD\uD843\uDFFE\uD843\uDFFF", -457487557, -697348811);
assertBucketEquals("string", null, 0, 0);
assertBucketEquals("string", "", 0, -965378730);
assertBucketEquals("string", "test string", -318923937, -138301454);
// 3-byte UTF-8 sequences (in Basic Plane, i.e. Plane 0)
assertBucketEquals("string", "\u5f3a\u5927\u7684Trino\u5f15\u64ce", 1688501507, 1784416344);
// 4 code points: 20FFC - 20FFF. 4-byte UTF-8 sequences in Supplementary Plane 2
assertBucketEquals("string", "\uD843\uDFFC\uD843\uDFFD\uD843\uDFFE\uD843\uDFFF", -1810797254, -697348811);
assertBucketEquals("date", null, 0, 0);
assertBucketEquals("date", Date.valueOf("1970-01-01"), 0, 1362653161);
assertBucketEquals("date", Date.valueOf("2015-11-19"), 16758, 8542395);
assertBucketEquals("date", Date.valueOf("1950-11-19"), -6983, -431619185);
for (BucketingVersion version : BucketingVersion.values()) {
List<TypeInfo> typeInfos = ImmutableList.of(timestampTypeInfo);
assertThatThrownBy(() -> version.getBucketHashCode(typeInfos, new Object[] { 0 })).hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
TimestampType timestampType = createTimestampType(3);
BlockBuilder builder = timestampType.createBlockBuilder(null, 1);
timestampType.writeLong(builder, 0);
Page page = new Page(builder.build());
assertThatThrownBy(() -> version.getBucketHashCode(typeInfos, page, 0)).hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
}
assertBucketEquals("array<double>", null, 0, 0);
assertBucketEquals("array<boolean>", ImmutableList.of(), 0, 0);
assertBucketEquals("array<smallint>", ImmutableList.of((short) 5, (short) 8, (short) 13), 5066, -905011156);
assertBucketEquals("array<string>", ImmutableList.of("test1", "test2", "test3", "test4"), 957612994, 1305539282);
assertBucketEquals("array<array<bigint>>", ImmutableList.of(ImmutableList.of(10L, 20L), ImmutableList.of(-10L, -20L), asList((Object) null)), 326368, 611324477);
assertBucketEquals("map<float,date>", null, 0, 0);
assertBucketEquals("map<double,timestamp>", ImmutableMap.of(), 0, 0);
assertBucketEquals("map<string,bigint>", ImmutableMap.of("key", 123L, "key2", 123456789L, "key3", -123456L), 127880789, -1910999650);
assertBucketEquals("map<array<double>,map<int,string>>", ImmutableMap.of(ImmutableList.of(12.3, 45.7), ImmutableMap.of(123, "test99")), -34001111, -1565874874);
// multiple bucketing columns
assertBucketEquals(ImmutableList.of("float", "array<smallint>", "map<string,bigint>"), ImmutableList.of(12.34F, ImmutableList.of((short) 5, (short) 8, (short) 13), ImmutableMap.of("key", 123L)), 95411006, 932898434);
assertBucketEquals(ImmutableList.of("double", "array<smallint>", "boolean", "map<string,bigint>", "tinyint"), asList(null, ImmutableList.of((short) 5, (short) 8, (short) 13), null, ImmutableMap.of("key", 123L), null), 154207826, -1120812524);
}
use of io.trino.plugin.hive.util.HiveBucketing.BucketingVersion in project trino by trinodb.
the class TestHivePartitionedBucketFunction method testMultiplePartitions.
@Test(dataProvider = "hiveBucketingVersion")
public void testMultiplePartitions(BucketingVersion hiveBucketingVersion) {
int numValues = 1024;
int numBuckets = 10;
Block bucketColumn = createLongSequenceBlockWithNull(numValues);
Page bucketedColumnPage = new Page(bucketColumn);
BucketFunction hiveBucketFunction = bucketFunction(hiveBucketingVersion, numBuckets, ImmutableList.of(HIVE_LONG));
int numPartitions = 8;
List<Long> partitionValues = new ArrayList<>();
for (int i = 0; i < numPartitions - 1; i++) {
partitionValues.addAll(Collections.nCopies(numValues / numPartitions, i * 348349L));
}
partitionValues.addAll(Collections.nCopies(numValues / numPartitions, null));
Block partitionColumn = createLongsBlock(partitionValues);
Page page = new Page(bucketColumn, partitionColumn);
Map<Long, HashMultimap<Integer, Integer>> partitionedBucketPositions = new HashMap<>();
for (int i = 0; i < numValues; i++) {
int hiveBucket = hiveBucketFunction.getBucket(bucketedColumnPage, i);
Long hivePartition = partitionValues.get(i);
// record list of positions for each combination of hive partition and bucket
partitionedBucketPositions.computeIfAbsent(hivePartition, ignored -> HashMultimap.create()).put(hiveBucket, i);
}
BucketFunction hivePartitionedBucketFunction = partitionedBucketFunction(hiveBucketingVersion, numBuckets, ImmutableList.of(HIVE_LONG), ImmutableList.of(BIGINT), 4000);
// All positions of a hive partition and bucket should hash to the same partitioned bucket
for (Map.Entry<Long, HashMultimap<Integer, Integer>> partitionEntry : partitionedBucketPositions.entrySet()) {
for (Map.Entry<Integer, Collection<Integer>> entry : partitionEntry.getValue().asMap().entrySet()) {
assertBucketCount(hivePartitionedBucketFunction, page, entry.getValue(), 1);
}
}
assertBucketCount(hivePartitionedBucketFunction, page, IntStream.range(0, numValues).boxed().collect(toImmutableList()), numBuckets * numPartitions);
}
use of io.trino.plugin.hive.util.HiveBucketing.BucketingVersion in project trino by trinodb.
the class BackgroundHiveSplitLoader method loadPartition.
private ListenableFuture<Void> loadPartition(HivePartitionMetadata partition) throws IOException {
HivePartition hivePartition = partition.getHivePartition();
String partitionName = hivePartition.getPartitionId();
Properties schema = getPartitionSchema(table, partition.getPartition());
List<HivePartitionKey> partitionKeys = getPartitionKeys(table, partition.getPartition());
TupleDomain<HiveColumnHandle> effectivePredicate = compactEffectivePredicate.transformKeys(HiveColumnHandle.class::cast);
BooleanSupplier partitionMatchSupplier = createPartitionMatchSupplier(dynamicFilter, hivePartition, getPartitionKeyColumnHandles(table, typeManager));
if (!partitionMatchSupplier.getAsBoolean()) {
// Avoid listing files and creating splits from a partition if it has been pruned due to dynamic filters
return COMPLETED_FUTURE;
}
Path path = new Path(getPartitionLocation(table, partition.getPartition()));
Configuration configuration = hdfsEnvironment.getConfiguration(hdfsContext, path);
InputFormat<?, ?> inputFormat = getInputFormat(configuration, schema, false);
FileSystem fs = hdfsEnvironment.getFileSystem(hdfsContext, path);
boolean s3SelectPushdownEnabled = shouldEnablePushdownForTable(session, table, path.toString(), partition.getPartition());
// S3 Select pushdown works at the granularity of individual S3 objects,
// therefore we must not split files when it is enabled.
// Skip header / footer lines are not splittable except for a special case when skip.header.line.count=1
boolean splittable = !s3SelectPushdownEnabled && getFooterCount(schema) == 0 && getHeaderCount(schema) <= 1;
if (inputFormat instanceof SymlinkTextInputFormat) {
if (tableBucketInfo.isPresent()) {
throw new TrinoException(NOT_SUPPORTED, "Bucketed table in SymlinkTextInputFormat is not yet supported");
}
InputFormat<?, ?> targetInputFormat = getInputFormat(configuration, schema, true);
List<Path> targetPaths = hdfsEnvironment.doAs(hdfsContext.getIdentity(), () -> getTargetPathsFromSymlink(fs, path));
Set<Path> parents = targetPaths.stream().map(Path::getParent).distinct().collect(toImmutableSet());
if (optimizeSymlinkListing && parents.size() == 1 && !recursiveDirWalkerEnabled) {
Optional<Iterator<InternalHiveSplit>> manifestFileIterator = buildManifestFileIterator(targetInputFormat, partitionName, schema, partitionKeys, effectivePredicate, partitionMatchSupplier, s3SelectPushdownEnabled, partition.getTableToPartitionMapping(), getOnlyElement(parents), targetPaths, splittable);
if (manifestFileIterator.isPresent()) {
fileIterators.addLast(manifestFileIterator.get());
return COMPLETED_FUTURE;
}
}
return createHiveSymlinkSplits(partitionName, targetInputFormat, schema, partitionKeys, effectivePredicate, partitionMatchSupplier, s3SelectPushdownEnabled, partition.getTableToPartitionMapping(), targetPaths);
}
Optional<BucketConversion> bucketConversion = Optional.empty();
boolean bucketConversionRequiresWorkerParticipation = false;
if (partition.getPartition().isPresent()) {
Optional<HiveBucketProperty> partitionBucketProperty = partition.getPartition().get().getStorage().getBucketProperty();
if (tableBucketInfo.isPresent() && partitionBucketProperty.isPresent()) {
int readBucketCount = tableBucketInfo.get().getReadBucketCount();
// TODO can partition's bucketing_version be different from table's?
BucketingVersion bucketingVersion = partitionBucketProperty.get().getBucketingVersion();
int partitionBucketCount = partitionBucketProperty.get().getBucketCount();
// Here, it's just trying to see if its needs the BucketConversion.
if (readBucketCount != partitionBucketCount) {
bucketConversion = Optional.of(new BucketConversion(bucketingVersion, readBucketCount, partitionBucketCount, tableBucketInfo.get().getBucketColumns()));
if (readBucketCount > partitionBucketCount) {
bucketConversionRequiresWorkerParticipation = true;
}
}
}
}
Optional<BucketValidation> bucketValidation = Optional.empty();
if (isValidateBucketing(session) && tableBucketInfo.isPresent()) {
BucketSplitInfo info = tableBucketInfo.get();
bucketValidation = Optional.of(new BucketValidation(info.getBucketingVersion(), info.getTableBucketCount(), info.getBucketColumns()));
}
InternalHiveSplitFactory splitFactory = new InternalHiveSplitFactory(fs, partitionName, inputFormat, schema, partitionKeys, effectivePredicate, partitionMatchSupplier, partition.getTableToPartitionMapping(), bucketConversionRequiresWorkerParticipation ? bucketConversion : Optional.empty(), bucketValidation, getMaxInitialSplitSize(session), isForceLocalScheduling(session), s3SelectPushdownEnabled, transaction, maxSplitFileSize);
// on the input format to obtain file splits.
if (shouldUseFileSplitsFromInputFormat(inputFormat)) {
if (tableBucketInfo.isPresent()) {
throw new TrinoException(NOT_SUPPORTED, "Trino cannot read bucketed partition in an input format with UseFileSplitsFromInputFormat annotation: " + inputFormat.getClass().getSimpleName());
}
if (AcidUtils.isTransactionalTable(table.getParameters())) {
throw new TrinoException(NOT_SUPPORTED, "Hive transactional tables in an input format with UseFileSplitsFromInputFormat annotation are not supported: " + inputFormat.getClass().getSimpleName());
}
JobConf jobConf = toJobConf(configuration);
FileInputFormat.setInputPaths(jobConf, path);
// Pass SerDes and Table parameters into input format configuration
fromProperties(schema).forEach(jobConf::set);
InputSplit[] splits = hdfsEnvironment.doAs(hdfsContext.getIdentity(), () -> inputFormat.getSplits(jobConf, 0));
return addSplitsToSource(splits, splitFactory);
}
List<Path> readPaths;
List<HdfsFileStatusWithId> fileStatusOriginalFiles = ImmutableList.of();
AcidInfo.Builder acidInfoBuilder = AcidInfo.builder(path);
boolean isFullAcid = AcidUtils.isFullAcidTable(table.getParameters());
if (AcidUtils.isTransactionalTable(table.getParameters())) {
AcidUtils.Directory directory = hdfsEnvironment.doAs(hdfsContext.getIdentity(), () -> AcidUtils.getAcidState(path, configuration, validWriteIds.orElseThrow(() -> new IllegalStateException("No validWriteIds present")), false, true));
if (isFullAcid) {
// From Hive version >= 3.0, delta/base files will always have file '_orc_acid_version' with value >= '2'.
Path baseOrDeltaPath = directory.getBaseDirectory() != null ? directory.getBaseDirectory() : (directory.getCurrentDirectories().size() > 0 ? directory.getCurrentDirectories().get(0).getPath() : null);
if (baseOrDeltaPath != null && AcidUtils.OrcAcidVersion.getAcidVersionFromMetaFile(baseOrDeltaPath, fs) >= 2) {
// Trino cannot read ORC ACID tables with version < 2 (written by Hive older than 3.0)
// See https://github.com/trinodb/trino/issues/2790#issuecomment-591901728 for more context
// We perform initial version check based on _orc_acid_version file here.
// If we cannot verify the version (the _orc_acid_version file may not exist),
// we will do extra check based on ORC datafile metadata in OrcPageSourceFactory.
acidInfoBuilder.setOrcAcidVersionValidated(true);
}
}
readPaths = new ArrayList<>();
// base
if (directory.getBaseDirectory() != null) {
readPaths.add(directory.getBaseDirectory());
}
// delta directories
for (AcidUtils.ParsedDelta delta : directory.getCurrentDirectories()) {
if (!delta.isDeleteDelta()) {
readPaths.add(delta.getPath());
}
}
// Create a registry of delete_delta directories for the partition
for (AcidUtils.ParsedDelta delta : directory.getCurrentDirectories()) {
if (delta.isDeleteDelta()) {
if (!isFullAcid) {
throw new TrinoException(HIVE_BAD_DATA, format("Unexpected delete delta for a non full ACID table '%s'. Would be ignored by the reader: %s", table.getSchemaTableName(), delta.getPath()));
}
acidInfoBuilder.addDeleteDelta(delta.getPath());
}
}
// initialize original files status list if present
fileStatusOriginalFiles = directory.getOriginalFiles();
for (HdfsFileStatusWithId hdfsFileStatusWithId : fileStatusOriginalFiles) {
Path originalFilePath = hdfsFileStatusWithId.getFileStatus().getPath();
long originalFileLength = hdfsFileStatusWithId.getFileStatus().getLen();
if (originalFileLength == 0) {
continue;
}
// Hive requires "original" files of transactional tables to conform to the bucketed tables naming pattern, to match them with delete deltas.
int bucketId = getRequiredBucketNumber(originalFilePath);
acidInfoBuilder.addOriginalFile(originalFilePath, originalFileLength, bucketId);
}
} else {
// TODO https://github.com/trinodb/trino/issues/7603 - we should not referece acidInfoBuilder at allwhen we are not reading from non-ACID table
// no ACID; no further validation needed
acidInfoBuilder.setOrcAcidVersionValidated(true);
readPaths = ImmutableList.of(path);
}
// Bucketed partitions are fully loaded immediately since all files must be loaded to determine the file to bucket mapping
if (tableBucketInfo.isPresent()) {
// TODO document in addToQueue() that it is sufficient to hold on to last returned future
ListenableFuture<Void> lastResult = immediateVoidFuture();
for (Path readPath : readPaths) {
// list all files in the partition
List<LocatedFileStatus> files = new ArrayList<>();
try {
Iterators.addAll(files, new HiveFileIterator(table, readPath, fs, directoryLister, namenodeStats, FAIL, ignoreAbsentPartitions));
} catch (HiveFileIterator.NestedDirectoryNotAllowedException e) {
// Fail here to be on the safe side. This seems to be the same as what Hive does
throw new TrinoException(HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. Found sub-directory '%s' in bucket directory for partition: %s", table.getSchemaTableName(), e.getNestedDirectoryPath(), splitFactory.getPartitionName()));
}
Optional<AcidInfo> acidInfo = isFullAcid ? acidInfoBuilder.build() : Optional.empty();
lastResult = hiveSplitSource.addToQueue(getBucketedSplits(files, splitFactory, tableBucketInfo.get(), bucketConversion, splittable, acidInfo));
}
for (HdfsFileStatusWithId hdfsFileStatusWithId : fileStatusOriginalFiles) {
List<LocatedFileStatus> locatedFileStatuses = ImmutableList.of((LocatedFileStatus) hdfsFileStatusWithId.getFileStatus());
Optional<AcidInfo> acidInfo = isFullAcid ? Optional.of(acidInfoBuilder.buildWithRequiredOriginalFiles(getRequiredBucketNumber(hdfsFileStatusWithId.getFileStatus().getPath()))) : Optional.empty();
lastResult = hiveSplitSource.addToQueue(getBucketedSplits(locatedFileStatuses, splitFactory, tableBucketInfo.get(), bucketConversion, splittable, acidInfo));
}
return lastResult;
}
for (Path readPath : readPaths) {
Optional<AcidInfo> acidInfo = isFullAcid ? acidInfoBuilder.build() : Optional.empty();
fileIterators.addLast(createInternalHiveSplitIterator(readPath, fs, splitFactory, splittable, acidInfo));
}
if (!fileStatusOriginalFiles.isEmpty()) {
fileIterators.addLast(generateOriginalFilesSplits(splitFactory, fileStatusOriginalFiles, splittable, acidInfoBuilder, isFullAcid));
}
return COMPLETED_FUTURE;
}
use of io.trino.plugin.hive.util.HiveBucketing.BucketingVersion in project trino by trinodb.
the class HiveTableProperties method getBucketProperty.
public static Optional<HiveBucketProperty> getBucketProperty(Map<String, Object> tableProperties) {
List<String> bucketedBy = getBucketedBy(tableProperties);
List<SortingColumn> sortedBy = getSortedBy(tableProperties);
int bucketCount = (Integer) tableProperties.get(BUCKET_COUNT_PROPERTY);
if ((bucketedBy.isEmpty()) && (bucketCount == 0)) {
if (!sortedBy.isEmpty()) {
throw new TrinoException(INVALID_TABLE_PROPERTY, format("%s may be specified only when %s is specified", SORTED_BY_PROPERTY, BUCKETED_BY_PROPERTY));
}
return Optional.empty();
}
if (bucketCount < 0) {
throw new TrinoException(INVALID_TABLE_PROPERTY, format("%s must be greater than zero", BUCKET_COUNT_PROPERTY));
}
if (bucketedBy.isEmpty() || bucketCount == 0) {
throw new TrinoException(INVALID_TABLE_PROPERTY, format("%s and %s must be specified together", BUCKETED_BY_PROPERTY, BUCKET_COUNT_PROPERTY));
}
BucketingVersion bucketingVersion = getBucketingVersion(tableProperties);
return Optional.of(new HiveBucketProperty(bucketedBy, bucketingVersion, bucketCount, sortedBy));
}
Aggregations