Search in sources :

Example 21 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class HiveMetadata method getTableLayouts.

@Override
public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) {
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    HivePartitionResult hivePartitionResult;
    if (handle.getAnalyzePartitionValues().isPresent()) {
        verify(constraint.getSummary().isAll(), "There shouldn't be any constraint for ANALYZE operation");
        hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, handle.getAnalyzePartitionValues().get(), session);
    } else {
        hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    }
    Map<String, HiveColumnHandle> predicateColumns = hivePartitionResult.getEffectivePredicate().getDomains().get().keySet().stream().map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    Optional<HiveBucketHandle> hiveBucketHandle = hivePartitionResult.getBucketHandle();
    int virtualBucketCount = getVirtualBucketCount(session);
    if (!hivePartitionResult.getBucketHandle().isPresent() && virtualBucketCount > 0) {
        hiveBucketHandle = Optional.of(createVirtualBucketHandle(virtualBucketCount));
    }
    TupleDomain<Subfield> domainPredicate = hivePartitionResult.getEffectivePredicate().transform(HiveMetadata::toSubfield);
    Table table = metastore.getTable(getMetastoreContext(session), handle.getSchemaTableName().getSchemaName(), handle.getSchemaTableName().getTableName()).orElseThrow(() -> new TableNotFoundException(handle.getSchemaTableName()));
    return ImmutableList.of(new ConnectorTableLayoutResult(getTableLayout(session, new HiveTableLayoutHandle(handle.getSchemaTableName(), table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, TRUE_CONSTANT, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hiveBucketHandle, hivePartitionResult.getBucketFilter(), false, createTableLayoutString(session, handle.getSchemaTableName(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), TRUE_CONSTANT, domainPredicate), desiredColumns.map(columns -> columns.stream().map(column -> (HiveColumnHandle) column).collect(toImmutableSet())), false)), hivePartitionResult.getUnenforcedConstraint()));
}
Also used : SystemTable(com.facebook.presto.spi.SystemTable) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) Table(com.facebook.presto.hive.metastore.Table) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) Constraint(com.facebook.presto.spi.Constraint) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HiveBucketing.getHiveBucketHandle(com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle) Subfield(com.facebook.presto.common.Subfield) ColumnWithStructSubfield(com.facebook.presto.hive.ColumnEncryptionInformation.ColumnWithStructSubfield)

Example 22 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class HiveMetadata method getTableStatistics.

@Override
public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional<ConnectorTableLayoutHandle> tableLayoutHandle, List<ColumnHandle> columnHandles, Constraint<ColumnHandle> constraint) {
    if (!isStatisticsEnabled(session)) {
        return TableStatistics.empty();
    }
    if (!tableLayoutHandle.isPresent() || !((HiveTableLayoutHandle) tableLayoutHandle.get()).isPushdownFilterEnabled()) {
        Map<String, ColumnHandle> columns = columnHandles.stream().map(HiveColumnHandle.class::cast).filter(not(HiveColumnHandle::isHidden)).collect(toImmutableMap(HiveColumnHandle::getName, Function.identity()));
        Map<String, Type> columnTypes = columns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType()));
        List<HivePartition> partitions = partitionManager.getPartitions(metastore, tableHandle, constraint, session).getPartitions();
        return hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), columns, columnTypes, partitions);
    }
    verify(!constraint.predicate().isPresent());
    HiveTableLayoutHandle hiveLayoutHandle = (HiveTableLayoutHandle) tableLayoutHandle.get();
    Set<String> columnNames = columnHandles.stream().map(HiveColumnHandle.class::cast).map(HiveColumnHandle::getName).collect(toImmutableSet());
    Set<ColumnHandle> allColumnHandles = ImmutableSet.<ColumnHandle>builder().addAll(columnHandles).addAll(hiveLayoutHandle.getPredicateColumns().values().stream().filter(column -> !columnNames.contains(column.getName())).collect(toImmutableList())).build();
    Map<String, ColumnHandle> allColumns = Maps.uniqueIndex(allColumnHandles, column -> ((HiveColumnHandle) column).getName());
    Map<String, Type> allColumnTypes = allColumns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType()));
    Constraint<ColumnHandle> combinedConstraint = new Constraint<>(constraint.getSummary().intersect(hiveLayoutHandle.getDomainPredicate().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(allColumns::get)));
    SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session);
    RowExpression domainPredicate = rowExpressionService.getDomainTranslator().toPredicate(hiveLayoutHandle.getDomainPredicate().transform(subfield -> subfieldExtractor.toRowExpression(subfield, allColumnTypes.get(subfield.getRootName()))));
    RowExpression combinedPredicate = binaryExpression(SpecialFormExpression.Form.AND, ImmutableList.of(hiveLayoutHandle.getRemainingPredicate(), domainPredicate));
    List<HivePartition> partitions = partitionManager.getPartitions(metastore, tableHandle, combinedConstraint, session).getPartitions();
    TableStatistics tableStatistics = hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), allColumns, allColumnTypes, partitions);
    return filterStatsCalculatorService.filterStats(tableStatistics, combinedPredicate, session, ImmutableBiMap.copyOf(allColumns).inverse(), allColumnTypes);
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Statistics.createEmptyPartitionStatistics(com.facebook.presto.hive.metastore.Statistics.createEmptyPartitionStatistics) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FILE_SIZE_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME) HiveSessionProperties.isPreferManifestsToListFiles(com.facebook.presto.hive.HiveSessionProperties.isPreferManifestsToListFiles) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) HiveSessionProperties.isOfflineDataDebugModeEnabled(com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) DWRF_ENCRYPTION_PROVIDER(com.facebook.presto.hive.HiveTableProperties.DWRF_ENCRYPTION_PROVIDER) MATERIALIZED_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW) HiveSessionProperties.isOptimizedPartitionUpdateSerializationEnabled(com.facebook.presto.hive.HiveSessionProperties.isOptimizedPartitionUpdateSerializationEnabled) Statistics.fromComputedStatistics(com.facebook.presto.hive.metastore.Statistics.fromComputedStatistics) MAX_VALUE_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE_SIZE_IN_BYTES) HiveTableProperties.getPartitionedBy(com.facebook.presto.hive.HiveTableProperties.getPartitionedBy) Map(java.util.Map) LocalProperty(com.facebook.presto.spi.LocalProperty) SystemTable(com.facebook.presto.spi.SystemTable) ENGLISH(java.util.Locale.ENGLISH) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) PATH_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME) HivePrivilegeInfo.toHivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) Collectors.joining(java.util.stream.Collectors.joining) Stream(java.util.stream.Stream) MIN_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MIN_VALUE) NUMBER_OF_DISTINCT_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_DISTINCT_VALUES) CSV_SEPARATOR(com.facebook.presto.hive.HiveTableProperties.CSV_SEPARATOR) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Joiner(com.google.common.base.Joiner) ConnectorPartitioningHandle(com.facebook.presto.spi.connector.ConnectorPartitioningHandle) Table(com.facebook.presto.hive.metastore.Table) Database(com.facebook.presto.hive.metastore.Database) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) HIVE_BINARY(com.facebook.presto.hive.HiveType.HIVE_BINARY) FULLY_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.FULLY_MATERIALIZED) HiveBucketHandle.createVirtualBucketHandle(com.facebook.presto.hive.HiveBucketHandle.createVirtualBucketHandle) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveTableProperties.getBucketProperty(com.facebook.presto.hive.HiveTableProperties.getBucketProperty) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) TableStatisticType(com.facebook.presto.spi.statistics.TableStatisticType) Supplier(java.util.function.Supplier) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) HiveMaterializedViewUtils.viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(com.facebook.presto.hive.HiveMaterializedViewUtils.viewToBaseTableOnOuterJoinSideIndirectMappedPartitions) OptionalLong(java.util.OptionalLong) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) Lists(com.google.common.collect.Lists) MaterializedDataPredicates(com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ENCRYPT_COLUMNS(com.facebook.presto.hive.HiveTableProperties.ENCRYPT_COLUMNS) Functions(com.google.common.base.Functions) HiveWriterFactory.computeBucketedFileName(com.facebook.presto.hive.HiveWriterFactory.computeBucketedFileName) RESPECT_TABLE_FORMAT(com.facebook.presto.hive.HiveSessionProperties.RESPECT_TABLE_FORMAT) IOException(java.io.IOException) Domain(com.facebook.presto.common.predicate.Domain) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) HiveUtil.hiveColumnHandles(com.facebook.presto.hive.HiveUtil.hiveColumnHandles) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) TableLayoutFilterCoverage(com.facebook.presto.spi.TableLayoutFilterCoverage) HiveManifestUtils.getManifestSizeInBytes(com.facebook.presto.hive.HiveManifestUtils.getManifestSizeInBytes) CSV_QUOTE(com.facebook.presto.hive.HiveTableProperties.CSV_QUOTE) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) HiveAnalyzeProperties.getPartitionList(com.facebook.presto.hive.HiveAnalyzeProperties.getPartitionList) NUMBER_OF_TRUE_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_TRUE_VALUES) JsonCodec(com.facebook.airlift.json.JsonCodec) DWRF_ENCRYPTION_ALGORITHM(com.facebook.presto.hive.HiveTableProperties.DWRF_ENCRYPTION_ALGORITHM) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveUtil.encodeMaterializedViewData(com.facebook.presto.hive.HiveUtil.encodeMaterializedViewData) StandardFunctionResolution(com.facebook.presto.spi.function.StandardFunctionResolution) Privilege(com.facebook.presto.spi.security.Privilege) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) HiveSessionProperties.getTemporaryTableSchema(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ADD(com.facebook.presto.hive.metastore.Statistics.ReduceOperator.ADD) ConnectorTablePartitioning(com.facebook.presto.spi.ConnectorTablePartitioning) Collectors.toMap(java.util.stream.Collectors.toMap) MaterializedViewNotFoundException(com.facebook.presto.spi.MaterializedViewNotFoundException) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) PRESTO_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) SpecialFormExpression(com.facebook.presto.spi.relation.SpecialFormExpression) ImmutableSet(com.google.common.collect.ImmutableSet) HIVE_UNKNOWN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR) INVALID_SCHEMA_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) SCHEMA_NOT_EMPTY(com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) DwrfTableEncryptionProperties.forPerColumn(com.facebook.presto.hive.DwrfTableEncryptionProperties.forPerColumn) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) LogicalRowExpressions.binaryExpression(com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression) TOTAL_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.TOTAL_SIZE_IN_BYTES) IntStream(java.util.stream.IntStream) OVERWRITE(com.facebook.presto.hive.PartitionUpdate.UpdateMode.OVERWRITE) MapType(com.facebook.presto.common.type.MapType) HiveSessionProperties.isRespectTableFormat(com.facebook.presto.hive.HiveSessionProperties.isRespectTableFormat) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) CompletableFuture(java.util.concurrent.CompletableFuture) HiveUtil.verifyPartitionTypeSupported(com.facebook.presto.hive.HiveUtil.verifyPartitionTypeSupported) OptionalInt(java.util.OptionalInt) Function(java.util.function.Function) InMemoryRecordSet(com.facebook.presto.spi.InMemoryRecordSet) HashSet(java.util.HashSet) UNPARTITIONED_ID(com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID) HiveMaterializedViewUtils.getViewToBasePartitionMap(com.facebook.presto.hive.HiveMaterializedViewUtils.getViewToBasePartitionMap) OpenCSVSerde(org.apache.hadoop.hive.serde2.OpenCSVSerde) PREFERRED_ORDERING_COLUMNS(com.facebook.presto.hive.HiveTableProperties.PREFERRED_ORDERING_COLUMNS) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) ALREADY_EXISTS(com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS) SmileCodec(com.facebook.airlift.json.smile.SmileCodec) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) NoSuchElementException(java.util.NoSuchElementException) HiveTableProperties.getDwrfEncryptionAlgorithm(com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionAlgorithm) Type(com.facebook.presto.common.type.Type) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) USER(com.facebook.presto.spi.security.PrincipalType.USER) Storage(com.facebook.presto.hive.metastore.Storage) HivePartitioningHandle.createHiveCompatiblePartitioningHandle(com.facebook.presto.hive.HivePartitioningHandle.createHiveCompatiblePartitioningHandle) ROW_COUNT(com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) HiveBucketing.getHiveBucketHandle(com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle) HiveTableProperties.getOrcBloomFilterColumns(com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterColumns) MoreFutures.toCompletableFuture(com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveMaterializedViewUtils.validateMaterializedViewPartitionColumns(com.facebook.presto.hive.HiveMaterializedViewUtils.validateMaterializedViewPartitionColumns) HiveTableProperties.getOrcBloomFilterFpp(com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterFpp) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) HiveSessionProperties.getTemporaryTableStorageFormat(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableStorageFormat) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) SYNTHESIZED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED) AVRO_SCHEMA_URL_KEY(com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY) Comparator(java.util.Comparator) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) NUMBER_OF_NON_NULL_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_NON_NULL_VALUES) HiveSessionProperties.isParquetPushdownFilterEnabled(com.facebook.presto.hive.HiveSessionProperties.isParquetPushdownFilterEnabled) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) HiveSessionProperties.isStatisticsEnabled(com.facebook.presto.hive.HiveSessionProperties.isStatisticsEnabled) EXTERNAL_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) AVRO_SCHEMA_URL(com.facebook.presto.hive.HiveTableProperties.AVRO_SCHEMA_URL) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) HiveSessionProperties.getBucketFunctionTypeForExchange(com.facebook.presto.hive.HiveSessionProperties.getBucketFunctionTypeForExchange) HiveMaterializedViewUtils.getMaterializedDataPredicates(com.facebook.presto.hive.HiveMaterializedViewUtils.getMaterializedDataPredicates) HiveTableProperties.getCsvProperty(com.facebook.presto.hive.HiveTableProperties.getCsvProperty) Predicates.not(com.google.common.base.Predicates.not) NOT_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.NOT_MATERIALIZED) ConnectorMaterializedViewDefinition(com.facebook.presto.spi.ConnectorMaterializedViewDefinition) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) FileWriteInfo(com.facebook.presto.hive.PartitionUpdate.FileWriteInfo) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) PrivilegeInfo(com.facebook.presto.spi.security.PrivilegeInfo) TEMPORARY_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) HiveUtil.decodeMaterializedViewData(com.facebook.presto.hive.HiveUtil.decodeMaterializedViewData) HiveManifestUtils.updatePartitionMetadataWithFileNamesAndSizes(com.facebook.presto.hive.HiveManifestUtils.updatePartitionMetadataWithFileNamesAndSizes) HIVE_INVALID_METADATA(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) HIVE_STORAGE_FORMAT(com.facebook.presto.hive.HiveSessionProperties.HIVE_STORAGE_FORMAT) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TypeUtils.isNumericType(com.facebook.presto.common.type.TypeUtils.isNumericType) JsonCodec.jsonCodec(com.facebook.airlift.json.JsonCodec.jsonCodec) ConnectorSession(com.facebook.presto.spi.ConnectorSession) CSV_ESCAPE(com.facebook.presto.hive.HiveTableProperties.CSV_ESCAPE) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) TableStatisticsMetadata(com.facebook.presto.spi.statistics.TableStatisticsMetadata) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) HiveColumnHandle.updateRowIdHandle(com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle) HiveSessionProperties.getTemporaryTableCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableCompressionCodec) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) HiveSessionProperties.isShufflePartitionedColumnsForTableWriteEnabled(com.facebook.presto.hive.HiveSessionProperties.isShufflePartitionedColumnsForTableWriteEnabled) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) HiveUtil.getPartitionKeyColumnHandles(com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles) HiveSessionProperties.isCollectColumnStatisticsOnWrite(com.facebook.presto.hive.HiveSessionProperties.isCollectColumnStatisticsOnWrite) ENCRYPT_TABLE(com.facebook.presto.hive.HiveTableProperties.ENCRYPT_TABLE) HiveTableProperties.getPreferredOrderingColumns(com.facebook.presto.hive.HiveTableProperties.getPreferredOrderingColumns) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HiveStatisticsProvider(com.facebook.presto.hive.statistics.HiveStatisticsProvider) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) Builder(com.google.common.collect.ImmutableMap.Builder) ArrayList(java.util.ArrayList) SortingProperty(com.facebook.presto.spi.SortingProperty) HiveTableProperties.getDwrfEncryptionProvider(com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionProvider) DwrfTableEncryptionProperties.fromHiveTableProperties(com.facebook.presto.hive.DwrfTableEncryptionProperties.fromHiveTableProperties) HiveUtil.encodeViewData(com.facebook.presto.hive.HiveUtil.encodeViewData) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) HiveBucketFilter(com.facebook.presto.hive.HiveBucketing.HiveBucketFilter) HiveWriteUtils.checkTableIsWritable(com.facebook.presto.hive.HiveWriteUtils.checkTableIsWritable) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) ADMIN_ROLE_NAME(com.facebook.presto.hive.security.SqlStandardAccessControl.ADMIN_ROLE_NAME) ConnectorMetadataUpdateHandle(com.facebook.presto.spi.ConnectorMetadataUpdateHandle) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) COVERED(com.facebook.presto.spi.TableLayoutFilterCoverage.COVERED) HiveSessionProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat) HiveSessionProperties.isOptimizedMismatchedBucketCount(com.facebook.presto.hive.HiveSessionProperties.isOptimizedMismatchedBucketCount) Properties(java.util.Properties) HiveSessionProperties.getCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getCompressionCodec) HIVE_CONCURRENT_MODIFICATION_DETECTED(com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED) Constraint(com.facebook.presto.spi.Constraint) File(java.io.File) PRESTO_MATERIALIZED_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG) Streams.stream(com.google.common.collect.Streams.stream) ColumnWithStructSubfield(com.facebook.presto.hive.ColumnEncryptionInformation.ColumnWithStructSubfield) INVALID_VIEW(com.facebook.presto.spi.StandardErrorCode.INVALID_VIEW) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HiveTableProperties.getEncryptColumns(com.facebook.presto.hive.HiveTableProperties.getEncryptColumns) INVALID_ANALYZE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY) QueryId(com.facebook.presto.spi.QueryId) HiveTableProperties.getAvroSchemaUrl(com.facebook.presto.hive.HiveTableProperties.getAvroSchemaUrl) HiveMaterializedViewUtils.differenceDataPredicates(com.facebook.presto.hive.HiveMaterializedViewUtils.differenceDataPredicates) URL(java.net.URL) HiveTableProperties.getExternalLocation(com.facebook.presto.hive.HiveTableProperties.getExternalLocation) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) HiveSessionProperties.isUsePageFileForHiveUnsupportedType(com.facebook.presto.hive.HiveSessionProperties.isUsePageFileForHiveUnsupportedType) FILE_MODIFIED_TIME_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME) HiveSessionProperties.isBucketExecutionEnabled(com.facebook.presto.hive.HiveSessionProperties.isBucketExecutionEnabled) Iterables.concat(com.google.common.collect.Iterables.concat) HIVE_COLUMN_ORDER_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) Splitter(com.google.common.base.Splitter) Collectors.toSet(java.util.stream.Collectors.toSet) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) HiveSessionProperties.isSortedWriteToTempPathEnabled(com.facebook.presto.hive.HiveSessionProperties.isSortedWriteToTempPathEnabled) ORC_BLOOM_FILTER_FPP(com.facebook.presto.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP) ImmutableMap(com.google.common.collect.ImmutableMap) HivePartitioningHandle.createPrestoNativePartitioningHandle(com.facebook.presto.hive.HivePartitioningHandle.createPrestoNativePartitioningHandle) NEW(com.facebook.presto.hive.PartitionUpdate.UpdateMode.NEW) NOT_COVERED(com.facebook.presto.spi.TableLayoutFilterCoverage.NOT_COVERED) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) MAX_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE) HiveSessionProperties.isStreamingAggregationEnabled(com.facebook.presto.hive.HiveSessionProperties.isStreamingAggregationEnabled) Sets(com.google.common.collect.Sets) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) RoleGrant(com.facebook.presto.spi.security.RoleGrant) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) HIVE_TIMEZONE_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_TIMEZONE_MISMATCH) ORC_BLOOM_FILTER_COLUMNS(com.facebook.presto.hive.HiveTableProperties.ORC_BLOOM_FILTER_COLUMNS) TOO_MANY_PARTITIONS_MISSING(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.TOO_MANY_PARTITIONS_MISSING) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) PRESTO_NATIVE(com.facebook.presto.hive.BucketFunctionType.PRESTO_NATIVE) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) FilterStatsCalculatorService(com.facebook.presto.spi.plan.FilterStatsCalculatorService) HiveWriterFactory.getFileExtension(com.facebook.presto.hive.HiveWriterFactory.getFileExtension) MetastoreUtil.getProtectMode(com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode) HiveSessionProperties.isSortedWritingEnabled(com.facebook.presto.hive.HiveSessionProperties.isSortedWritingEnabled) Column(com.facebook.presto.hive.metastore.Column) ColumnStatisticMetadata(com.facebook.presto.spi.statistics.ColumnStatisticMetadata) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) PrestoException(com.facebook.presto.spi.PrestoException) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Verify.verify(com.google.common.base.Verify.verify) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getVirtualBucketCount(com.facebook.presto.hive.HiveSessionProperties.getVirtualBucketCount) Objects.requireNonNull(java.util.Objects.requireNonNull) MetastoreUtil.verifyOnline(com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline) Suppliers(com.google.common.base.Suppliers) HiveSessionProperties.isFileRenamingEnabled(com.facebook.presto.hive.HiveSessionProperties.isFileRenamingEnabled) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) VerifyException(com.google.common.base.VerifyException) GrantInfo(com.facebook.presto.spi.security.GrantInfo) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) NOT_APPLICABLE(com.facebook.presto.spi.TableLayoutFilterCoverage.NOT_APPLICABLE) MalformedURLException(java.net.MalformedURLException) HIVE_UNSUPPORTED_ENCRYPTION_OPERATION(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_ENCRYPTION_OPERATION) Statistics.reduce(com.facebook.presto.hive.metastore.Statistics.reduce) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) EXTERNAL_LOCATION_PROPERTY(com.facebook.presto.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY) HiveSessionProperties.isCreateEmptyBucketFiles(com.facebook.presto.hive.HiveSessionProperties.isCreateEmptyBucketFiles) VIEW_STORAGE_FORMAT(com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) HiveUtil.deserializeZstdCompressed(com.facebook.presto.hive.HiveUtil.deserializeZstdCompressed) Maps(com.google.common.collect.Maps) ThriftMetastoreUtil.listEnabledPrincipals(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.listEnabledPrincipals) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) VIRTUAL_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) PARTIALLY_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.PARTIALLY_MATERIALIZED) UUID.randomUUID(java.util.UUID.randomUUID) ThriftMetastoreUtil(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) HIVE_PARTITION_READ_ONLY(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_READ_ONLY) HiveStorageFormat.values(com.facebook.presto.hive.HiveStorageFormat.values) APPEND(com.facebook.presto.hive.PartitionUpdate.UpdateMode.APPEND) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) MetastoreUtil.isUserDefinedTypeEncodingEnabled(com.facebook.presto.hive.metastore.MetastoreUtil.isUserDefinedTypeEncodingEnabled) HiveUtil.decodeViewData(com.facebook.presto.hive.HiveUtil.decodeViewData) MaterializedViewStatus(com.facebook.presto.spi.MaterializedViewStatus) VisibleForTesting(com.google.common.annotations.VisibleForTesting) HiveSessionProperties.getOrcCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getOrcCompressionCodec) Block(com.facebook.presto.common.block.Block) Statistics.createComputedStatisticsToPartitionMap(com.facebook.presto.hive.metastore.Statistics.createComputedStatisticsToPartitionMap) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Constraint(com.facebook.presto.spi.Constraint) RowExpression(com.facebook.presto.spi.relation.RowExpression) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) TableStatisticType(com.facebook.presto.spi.statistics.TableStatisticType) RowType(com.facebook.presto.common.type.RowType) MapType(com.facebook.presto.common.type.MapType) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) Type(com.facebook.presto.common.type.Type) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) TypeUtils.isNumericType(com.facebook.presto.common.type.TypeUtils.isNumericType) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) ArrayType(com.facebook.presto.common.type.ArrayType) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) HiveSessionProperties.isUsePageFileForHiveUnsupportedType(com.facebook.presto.hive.HiveSessionProperties.isUsePageFileForHiveUnsupportedType) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) Map(java.util.Map) Collectors.toMap(java.util.stream.Collectors.toMap) HiveMaterializedViewUtils.getViewToBasePartitionMap(com.facebook.presto.hive.HiveMaterializedViewUtils.getViewToBasePartitionMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Statistics.createComputedStatisticsToPartitionMap(com.facebook.presto.hive.metastore.Statistics.createComputedStatisticsToPartitionMap)

Example 23 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class HivePageSourceProvider method createSelectivePageSource.

private static Optional<ConnectorPageSource> createSelectivePageSource(Set<HiveSelectivePageSourceFactory> selectivePageSourceFactories, Configuration configuration, ConnectorSession session, HiveSplit split, HiveTableLayoutHandle layout, List<HiveColumnHandle> columns, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, LoadingCache<RowExpressionCacheKey, RowExpression> rowExpressionCache, SplitContext splitContext, Optional<EncryptionInformation> encryptionInformation) {
    Set<HiveColumnHandle> interimColumns = ImmutableSet.<HiveColumnHandle>builder().addAll(layout.getPredicateColumns().values()).addAll(split.getBucketConversion().map(BucketConversion::getBucketColumnHandles).orElse(ImmutableList.of())).build();
    Set<String> columnNames = columns.stream().map(HiveColumnHandle::getName).collect(toImmutableSet());
    List<HiveColumnHandle> allColumns = ImmutableList.<HiveColumnHandle>builder().addAll(columns).addAll(interimColumns.stream().filter(column -> !columnNames.contains(column.getName())).collect(toImmutableList())).build();
    Path path = new Path(split.getPath());
    List<ColumnMapping> columnMappings = ColumnMapping.buildColumnMappings(split.getPartitionKeys(), allColumns, ImmutableList.of(), split.getTableToPartitionMapping(), path, split.getTableBucketNumber(), split.getFileSize(), split.getFileModifiedTime());
    Optional<BucketAdaptation> bucketAdaptation = split.getBucketConversion().map(conversion -> toBucketAdaptation(conversion, columnMappings, split.getTableBucketNumber(), mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex()));
    Map<Integer, String> prefilledValues = columnMappings.stream().filter(mapping -> mapping.getKind() == ColumnMappingKind.PREFILLED).collect(toImmutableMap(mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex(), ColumnMapping::getPrefilledValue));
    Map<Integer, HiveCoercer> coercers = columnMappings.stream().filter(mapping -> mapping.getCoercionFrom().isPresent()).collect(toImmutableMap(mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex(), mapping -> createCoercer(typeManager, mapping.getCoercionFrom().get(), mapping.getHiveColumnHandle().getHiveType())));
    List<Integer> outputColumns = columns.stream().map(HiveColumnHandle::getHiveColumnIndex).collect(toImmutableList());
    RowExpression optimizedRemainingPredicate = rowExpressionCache.getUnchecked(new RowExpressionCacheKey(layout.getRemainingPredicate(), session));
    if (shouldSkipBucket(layout, split, splitContext)) {
        return Optional.of(new HiveEmptySplitPageSource());
    }
    if (shouldSkipPartition(typeManager, layout, hiveStorageTimeZone, split, splitContext)) {
        return Optional.of(new HiveEmptySplitPageSource());
    }
    CacheQuota cacheQuota = generateCacheQuota(split);
    for (HiveSelectivePageSourceFactory pageSourceFactory : selectivePageSourceFactories) {
        Optional<? extends ConnectorPageSource> pageSource = pageSourceFactory.createPageSource(configuration, session, path, split.getStart(), split.getLength(), split.getFileSize(), split.getStorage(), toColumnHandles(columnMappings, true), prefilledValues, coercers, bucketAdaptation, outputColumns, splitContext.getDynamicFilterPredicate().map(filter -> filter.transform(handle -> new Subfield(((HiveColumnHandle) handle).getName())).intersect(layout.getDomainPredicate())).orElse(layout.getDomainPredicate()), optimizedRemainingPredicate, hiveStorageTimeZone, new HiveFileContext(splitContext.isCacheable(), cacheQuota, split.getExtraFileInfo().map(BinaryExtraHiveFileInfo::new), Optional.of(split.getFileSize()), split.getFileModifiedTime(), HiveSessionProperties.isVerboseRuntimeStatsEnabled(session)), encryptionInformation);
        if (pageSource.isPresent()) {
            return Optional.of(pageSource.get());
        }
    }
    return Optional.empty();
}
Also used : NestedField(com.facebook.presto.common.Subfield.NestedField) RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) LoadingCache(com.google.common.cache.LoadingCache) HiveColumnHandle.isPushedDownSubfield(com.facebook.presto.hive.HiveColumnHandle.isPushedDownSubfield) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) ColumnMapping.toColumnHandles(com.facebook.presto.hive.HivePageSourceProvider.ColumnMapping.toColumnHandles) AGGREGATED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.AGGREGATED) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) SplitContext(com.facebook.presto.spi.SplitContext) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) HiveBucketing.getHiveBucketFilter(com.facebook.presto.hive.HiveBucketing.getHiveBucketFilter) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) BucketConversion(com.facebook.presto.hive.HiveSplit.BucketConversion) ImmutableSet(com.google.common.collect.ImmutableSet) HIVE_UNKNOWN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR) NullableValue(com.facebook.presto.common.predicate.NullableValue) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ConnectorSession(com.facebook.presto.spi.ConnectorSession) CacheLoader(com.google.common.cache.CacheLoader) RecordCursor(com.facebook.presto.spi.RecordCursor) DataSize(io.airlift.units.DataSize) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) Optional(java.util.Optional) CacheBuilder(com.google.common.cache.CacheBuilder) HiveCoercer.createCoercer(com.facebook.presto.hive.HiveCoercer.createCoercer) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Column(com.facebook.presto.hive.metastore.Column) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) Function(java.util.function.Function) HiveSessionProperties.isUseRecordPageSourceForCustomSplit(com.facebook.presto.hive.HiveSessionProperties.isUseRecordPageSourceForCustomSplit) System.identityHashCode(java.lang.System.identityHashCode) Inject(javax.inject.Inject) HashSet(java.util.HashSet) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) TypeManager(com.facebook.presto.common.type.TypeManager) Objects.requireNonNull(java.util.Objects.requireNonNull) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Type(com.facebook.presto.common.type.Type) RowExpression(com.facebook.presto.spi.relation.RowExpression) Storage(com.facebook.presto.hive.metastore.Storage) Properties(java.util.Properties) PathElement(com.facebook.presto.common.Subfield.PathElement) HiveUtil.getPrefilledColumnValue(com.facebook.presto.hive.HiveUtil.getPrefilledColumnValue) MetastoreUtil.reconstructPartitionSchema(com.facebook.presto.hive.metastore.MetastoreUtil.reconstructPartitionSchema) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Domain(com.facebook.presto.common.predicate.Domain) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) OPTIMIZED(com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SYNTHESIZED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED) HiveUtil.parsePartitionValue(com.facebook.presto.hive.HiveUtil.parsePartitionValue) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) HiveUtil.shouldUseRecordReaderFromInputFormat(com.facebook.presto.hive.HiveUtil.shouldUseRecordReaderFromInputFormat) HiveColumnHandle.isPushedDownSubfield(com.facebook.presto.hive.HiveColumnHandle.isPushedDownSubfield) Subfield(com.facebook.presto.common.Subfield) Path(org.apache.hadoop.fs.Path) RowExpression(com.facebook.presto.spi.relation.RowExpression)

Example 24 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class HiveSplitManager method getPartitionSplitInfo.

private Map<String, PartitionSplitInfo> getPartitionSplitInfo(ConnectorSession session, SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, List<HivePartition> partitionBatch, Map<String, HiveColumnHandle> predicateColumns, Optional<Map<Subfield, Domain>> domains) {
    MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider());
    Map<String, Optional<Partition>> partitions = metastore.getPartitionsByNames(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId));
    Map<String, PartitionStatistics> partitionStatistics = ImmutableMap.of();
    if (domains.isPresent() && isPartitionStatisticsBasedOptimizationEnabled(session)) {
        partitionStatistics = metastore.getPartitionStatistics(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), partitionBatch.stream().map(HivePartition::getPartitionId).collect(toImmutableSet()));
    }
    Map<String, String> partitionNameToLocation = new HashMap<>();
    ImmutableMap.Builder<String, PartitionSplitInfo> partitionSplitInfoBuilder = ImmutableMap.builder();
    for (Map.Entry<String, Optional<Partition>> entry : partitions.entrySet()) {
        ImmutableSet.Builder<ColumnHandle> redundantColumnDomainsBuilder = ImmutableSet.builder();
        if (!entry.getValue().isPresent()) {
            throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Partition no longer exists: " + entry.getKey());
        }
        boolean pruned = false;
        if (partitionStatistics.containsKey(entry.getKey())) {
            Map<String, HiveColumnStatistics> columnStatistics = partitionStatistics.get(entry.getKey()).getColumnStatistics();
            for (Map.Entry<String, HiveColumnHandle> predicateColumnEntry : predicateColumns.entrySet()) {
                if (columnStatistics.containsKey(predicateColumnEntry.getKey())) {
                    Optional<ValueSet> columnsStatisticsValueSet = getColumnStatisticsValueSet(columnStatistics.get(predicateColumnEntry.getKey()), predicateColumnEntry.getValue().getHiveType());
                    Subfield subfield = new Subfield(predicateColumnEntry.getKey());
                    if (columnsStatisticsValueSet.isPresent() && domains.get().containsKey(subfield)) {
                        ValueSet columnPredicateValueSet = domains.get().get(subfield).getValues();
                        if (!columnPredicateValueSet.overlaps(columnsStatisticsValueSet.get())) {
                            pruned = true;
                            break;
                        }
                        if (columnPredicateValueSet.contains(columnsStatisticsValueSet.get())) {
                            redundantColumnDomainsBuilder.add(predicateColumnEntry.getValue());
                        }
                    }
                }
            }
        }
        if (!pruned) {
            partitionNameToLocation.put(entry.getKey(), entry.getValue().get().getStorage().getLocation());
        }
        partitionSplitInfoBuilder.put(entry.getKey(), new PartitionSplitInfo(entry.getValue().get(), pruned, redundantColumnDomainsBuilder.build()));
    }
    metastore.setPartitionLeases(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), partitionNameToLocation, getLeaseDuration(session));
    return partitionSplitInfoBuilder.build();
}
Also used : HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) ImmutableSet(com.google.common.collect.ImmutableSet) ValueSet(com.facebook.presto.common.predicate.ValueSet) Subfield(com.facebook.presto.common.Subfield) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HiveColumnHandle.isPathColumnHandle(com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle) Optional(java.util.Optional) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 25 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class OrcSelectivePageSourceFactory method toFilterFunctions.

/**
 * Split filter expression into groups of conjuncts that depend on the same set of inputs,
 * then compile each group into FilterFunction.
 */
private static List<FilterFunction> toFilterFunctions(RowExpression filter, Optional<BucketAdapter> bucketAdapter, ConnectorSession session, DeterminismEvaluator determinismEvaluator, PredicateCompiler predicateCompiler) {
    ImmutableList.Builder<FilterFunction> filterFunctions = ImmutableList.builder();
    bucketAdapter.map(predicate -> new FilterFunction(session.getSqlFunctionProperties(), true, predicate)).ifPresent(filterFunctions::add);
    if (TRUE_CONSTANT.equals(filter)) {
        return filterFunctions.build();
    }
    DynamicFilterExtractResult extractDynamicFilterResult = extractDynamicFilters(filter);
    // dynamic filter will be added through subfield pushdown
    filter = and(extractDynamicFilterResult.getStaticConjuncts());
    if (!isAdaptiveFilterReorderingEnabled(session)) {
        filterFunctions.add(new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), filter).get()));
        return filterFunctions.build();
    }
    List<RowExpression> conjuncts = extractConjuncts(filter);
    if (conjuncts.size() == 1) {
        filterFunctions.add(new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), filter).get()));
        return filterFunctions.build();
    }
    // Use LinkedHashMap to preserve user-specified order of conjuncts. This will be the initial order in which filters are applied.
    Map<Set<Integer>, List<RowExpression>> inputsToConjuncts = new LinkedHashMap<>();
    for (RowExpression conjunct : conjuncts) {
        inputsToConjuncts.computeIfAbsent(extractInputs(conjunct), k -> new ArrayList<>()).add(conjunct);
    }
    inputsToConjuncts.values().stream().map(expressions -> binaryExpression(AND, expressions)).map(predicate -> new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(predicate), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), predicate).get())).forEach(filterFunctions::add);
    return filterFunctions.build();
}
Also used : LogicalRowExpressions.extractConjuncts(com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts) Page(com.facebook.presto.common.Page) HiveCoercer(com.facebook.presto.hive.HiveCoercer) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FixedPageSource(com.facebook.presto.spi.FixedPageSource) AGGREGATED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.AGGREGATED) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Predicate(com.facebook.presto.common.relation.Predicate) HiveSessionProperties.isOrcBloomFiltersEnabled(com.facebook.presto.hive.HiveSessionProperties.isOrcBloomFiltersEnabled) OrcDataSource(com.facebook.presto.orc.OrcDataSource) FileFormatDataSourceStats(com.facebook.presto.hive.FileFormatDataSourceStats) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) HiveSessionProperties.isOrcZstdJniDecompressionEnabled(com.facebook.presto.hive.HiveSessionProperties.isOrcZstdJniDecompressionEnabled) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) HiveFileContext(com.facebook.presto.hive.HiveFileContext) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ORC(com.facebook.presto.orc.OrcEncoding.ORC) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) DefaultRowExpressionTraversalVisitor(com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) DwrfKeyProvider(com.facebook.presto.orc.DwrfKeyProvider) OrcReaderOptions(com.facebook.presto.orc.OrcReaderOptions) BucketAdaptation(com.facebook.presto.hive.BucketAdaptation) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) DwrfEncryptionProvider(com.facebook.presto.orc.DwrfEncryptionProvider) OrcDataSourceId(com.facebook.presto.orc.OrcDataSourceId) HIVE_INVALID_BUCKET_FILES(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES) OrcSelectiveRecordReader(com.facebook.presto.orc.OrcSelectiveRecordReader) SubfieldExtractor(com.facebook.presto.hive.SubfieldExtractor) LogicalRowExpressions.and(com.facebook.presto.expressions.LogicalRowExpressions.and) IOException(java.io.IOException) HiveBucketing.getHiveBucket(com.facebook.presto.hive.HiveBucketing.getHiveBucket) OrcReader(com.facebook.presto.orc.OrcReader) DynamicFilters.extractDynamicFilters(com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters) HdfsEnvironment(com.facebook.presto.hive.HdfsEnvironment) TupleDomainOrcPredicate(com.facebook.presto.orc.TupleDomainOrcPredicate) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) HiveSessionProperties.getOrcTinyStripeThreshold(com.facebook.presto.hive.HiveSessionProperties.getOrcTinyStripeThreshold) StandardFunctionResolution(com.facebook.presto.spi.function.StandardFunctionResolution) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) HiveSessionProperties.getOrcLazyReadSmallRanges(com.facebook.presto.hive.HiveSessionProperties.getOrcLazyReadSmallRanges) AND(com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND) Path(org.apache.hadoop.fs.Path) EncryptionInformation(com.facebook.presto.hive.EncryptionInformation) CallExpression(com.facebook.presto.spi.relation.CallExpression) SpecialFormExpression(com.facebook.presto.spi.relation.SpecialFormExpression) BiMap(com.google.common.collect.BiMap) HiveClientConfig(com.facebook.presto.hive.HiveClientConfig) StripeMetadataSourceFactory(com.facebook.presto.orc.StripeMetadataSourceFactory) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) INITIAL_BATCH_SIZE(com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE) OrcPredicate(com.facebook.presto.orc.OrcPredicate) FileNotFoundException(java.io.FileNotFoundException) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) HiveSessionProperties.isAdaptiveFilterReorderingEnabled(com.facebook.presto.hive.HiveSessionProperties.isAdaptiveFilterReorderingEnabled) String.format(java.lang.String.format) DataSize(io.airlift.units.DataSize) List(java.util.List) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) HiveOrcAggregatedMemoryContext(com.facebook.presto.hive.HiveOrcAggregatedMemoryContext) HiveSelectivePageSourceFactory(com.facebook.presto.hive.HiveSelectivePageSourceFactory) Optional(java.util.Optional) LogicalRowExpressions.binaryExpression(com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression) HiveColumnHandle(com.facebook.presto.hive.HiveColumnHandle) PredicateCompiler(com.facebook.presto.spi.relation.PredicateCompiler) InputReferenceExpression(com.facebook.presto.spi.relation.InputReferenceExpression) IntStream(java.util.stream.IntStream) DynamicFilterExtractResult(com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult) DeterminismEvaluator(com.facebook.presto.spi.relation.DeterminismEvaluator) HiveSessionProperties.getOrcMaxMergeDistance(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxMergeDistance) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) HiveType(com.facebook.presto.hive.HiveType) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) Function(java.util.function.Function) HIVE_CANNOT_OPEN_SPLIT(com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT) HIVE_MISSING_DATA(com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA) HiveSessionProperties.getOrcMaxReadBlockSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxReadBlockSize) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Inject(javax.inject.Inject) HashSet(java.util.HashSet) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) UncheckedExecutionException(com.google.common.util.concurrent.UncheckedExecutionException) FilterFunction(com.facebook.presto.common.predicate.FilterFunction) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getOrcMaxBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcFileTailSource(com.facebook.presto.orc.cache.OrcFileTailSource) Type(com.facebook.presto.common.type.Type) RowExpression(com.facebook.presto.spi.relation.RowExpression) Storage(com.facebook.presto.hive.metastore.Storage) OrcAggregatedMemoryContext(com.facebook.presto.orc.OrcAggregatedMemoryContext) OrcEncoding(com.facebook.presto.orc.OrcEncoding) HiveSessionProperties.getOrcStreamBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize) NO_ENCRYPTION(com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION) Maps(com.google.common.collect.Maps) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) Consumer(java.util.function.Consumer) HiveUtil.typedPartitionKey(com.facebook.presto.hive.HiveUtil.typedPartitionKey) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) HiveUtil.getPhysicalHiveColumnHandles(com.facebook.presto.hive.HiveUtil.getPhysicalHiveColumnHandles) ImmutableBiMap.toImmutableBiMap(com.google.common.collect.ImmutableBiMap.toImmutableBiMap) RowExpressionNodeInliner.replaceExpression(com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression) FilterFunction(com.facebook.presto.common.predicate.FilterFunction) Set(java.util.Set) ImmutableSet(com.google.common.collect.ImmutableSet) HashSet(java.util.HashSet) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) RowExpression(com.facebook.presto.spi.relation.RowExpression) LinkedHashMap(java.util.LinkedHashMap) DynamicFilterExtractResult(com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList)

Aggregations

Subfield (com.facebook.presto.common.Subfield)54 ImmutableMap (com.google.common.collect.ImmutableMap)27 Map (java.util.Map)27 ImmutableList (com.google.common.collect.ImmutableList)25 List (java.util.List)24 TupleDomainFilter (com.facebook.presto.common.predicate.TupleDomainFilter)22 Type (com.facebook.presto.common.type.Type)21 ArrayList (java.util.ArrayList)21 Optional (java.util.Optional)20 Test (org.testng.annotations.Test)19 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)18 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)18 Collectors.toList (java.util.stream.Collectors.toList)12 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)11 ColumnHandle (com.facebook.presto.spi.ColumnHandle)11 String.format (java.lang.String.format)11 Set (java.util.Set)11 Domain (com.facebook.presto.common.predicate.Domain)10 CharType (com.facebook.presto.common.type.CharType)10 DecimalType (com.facebook.presto.common.type.DecimalType)10