Search in sources :

Example 56 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveMetadata method createMaterializedView.

@Override
public void createMaterializedView(ConnectorSession session, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) {
    if (isExternalTable(viewMetadata.getProperties())) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, "Specifying external location for materialized view is not supported.");
    }
    Table basicTable = prepareTable(session, viewMetadata, MATERIALIZED_VIEW);
    viewDefinition = new ConnectorMaterializedViewDefinition(viewDefinition.getOriginalSql(), viewDefinition.getSchema(), viewDefinition.getTable(), viewDefinition.getBaseTables(), viewDefinition.getOwner(), viewDefinition.getColumnMappings(), viewDefinition.getBaseTablesOnOuterJoinSide(), Optional.of(getPartitionedBy(viewMetadata.getProperties())));
    Map<String, String> parameters = ImmutableMap.<String, String>builder().putAll(basicTable.getParameters()).put(PRESTO_MATERIALIZED_VIEW_FLAG, "true").build();
    Table viewTable = Table.builder(basicTable).setParameters(parameters).setViewOriginalText(Optional.of(encodeMaterializedViewData(MATERIALIZED_VIEW_JSON_CODEC.toJson(viewDefinition)))).setViewExpandedText(Optional.of("/* Presto Materialized View */")).build();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    validateMaterializedViewPartitionColumns(metastore, metastoreContext, viewTable, viewDefinition);
    try {
        PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(viewTable.getOwner());
        metastore.createTable(session, viewTable, principalPrivileges, Optional.empty(), ignoreExisting, new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
    } catch (TableAlreadyExistsException e) {
        throw new MaterializedViewAlreadyExistsException(e.getTableName());
    }
}
Also used : SystemTable(com.facebook.presto.spi.SystemTable) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) Table(com.facebook.presto.hive.metastore.Table) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Statistics.createEmptyPartitionStatistics(com.facebook.presto.hive.metastore.Statistics.createEmptyPartitionStatistics) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) ConnectorMaterializedViewDefinition(com.facebook.presto.spi.ConnectorMaterializedViewDefinition) PrestoException(com.facebook.presto.spi.PrestoException)

Example 57 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveMetadata method getPreferredShuffleLayoutForInsert.

@Override
public Optional<ConnectorNewTableLayout> getPreferredShuffleLayoutForInsert(ConnectorSession session, ConnectorTableHandle tableHandle) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(table);
    if (hiveBucketHandle.isPresent()) {
        // and there is no additional preferred shuffle partitioning
        return Optional.empty();
    }
    if (!isShufflePartitionedColumnsForTableWriteEnabled(session) || table.getPartitionColumns().isEmpty()) {
        return Optional.empty();
    }
    // TODO: the shuffle partitioning could use a better hash function (instead of Hive bucket function)
    HivePartitioningHandle partitioningHandle = createHiveCompatiblePartitioningHandle(SHUFFLE_MAX_PARALLELISM_FOR_PARTITIONED_TABLE_WRITE, table.getPartitionColumns().stream().map(Column::getType).collect(toList()), OptionalInt.empty());
    List<String> partitionedBy = table.getPartitionColumns().stream().map(Column::getName).collect(toList());
    return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionedBy));
}
Also used : TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) SystemTable(com.facebook.presto.spi.SystemTable) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) Table(com.facebook.presto.hive.metastore.Table) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) DwrfTableEncryptionProperties.forPerColumn(com.facebook.presto.hive.DwrfTableEncryptionProperties.forPerColumn) Column(com.facebook.presto.hive.metastore.Column) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) HiveBucketing.getHiveBucketHandle(com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout)

Example 58 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveMetadata method getViews.

@Override
public Map<SchemaTableName, ConnectorViewDefinition> getViews(ConnectorSession session, SchemaTablePrefix prefix) {
    ImmutableMap.Builder<SchemaTableName, ConnectorViewDefinition> views = ImmutableMap.builder();
    List<SchemaTableName> tableNames;
    if (prefix.getTableName() != null) {
        tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    } else {
        tableNames = listViews(session, prefix.getSchemaName());
    }
    MetastoreContext metastoreContext = getMetastoreContext(session);
    for (SchemaTableName schemaTableName : tableNames) {
        Optional<Table> table = metastore.getTable(metastoreContext, schemaTableName.getSchemaName(), schemaTableName.getTableName());
        if (table.isPresent() && MetastoreUtil.isPrestoView(table.get())) {
            views.put(schemaTableName, new ConnectorViewDefinition(schemaTableName, Optional.ofNullable(table.get().getOwner()), decodeViewData(table.get().getViewOriginalText().get())));
        }
    }
    return views.build();
}
Also used : SystemTable(com.facebook.presto.spi.SystemTable) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) Table(com.facebook.presto.hive.metastore.Table) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition)

Example 59 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveMetadata method finishCreateTable.

@Override
public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics) {
    HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle;
    List<PartitionUpdate> partitionUpdates = getPartitionUpdates(session, fragments);
    Map<String, String> tableEncryptionParameters = ImmutableMap.of();
    Map<String, String> partitionEncryptionParameters = ImmutableMap.of();
    if (handle.getEncryptionInformation().isPresent()) {
        EncryptionInformation encryptionInformation = handle.getEncryptionInformation().get();
        if (encryptionInformation.getDwrfEncryptionMetadata().isPresent()) {
            if (handle.getPartitionedBy().isEmpty()) {
                tableEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            } else {
                partitionEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            }
        }
    }
    WriteInfo writeInfo = locationService.getQueryWriteInfo(handle.getLocationHandle());
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = buildTableObject(session.getQueryId(), handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), handle.getInputColumns(), handle.getTableStorageFormat(), handle.getPartitionedBy(), handle.getBucketProperty(), handle.getPreferredOrderingColumns(), ImmutableMap.<String, String>builder().putAll(handle.getAdditionalTableParameters()).putAll(tableEncryptionParameters).build(), writeInfo.getTargetPath(), MANAGED_TABLE, prestoVersion, metastoreContext);
    PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(handle.getTableOwner());
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    if (handle.getBucketProperty().isPresent() && isCreateEmptyBucketFiles(session)) {
        List<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(session, handle, table, partitionUpdates);
        // replace partitionUpdates before creating the zero-row files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        HdfsContext hdfsContext = new HdfsContext(session, table.getDatabaseName(), table.getTableName(), table.getStorage().getLocation(), true);
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(partitionObjectBuilder.buildPartitionObject(session, table, partitionUpdate, prestoVersion, partitionEncryptionParameters));
            zeroRowFileCreator.createFiles(session, hdfsContext, partitionUpdate.getWritePath(), getTargetFileNames(partitionUpdate.getFileWriteInfos()), getStorageFormat(partition, table), handle.getCompressionCodec(), getSchema(partition, table));
        }
    }
    Map<String, Type> columnTypes = handle.getInputColumns().stream().collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
    PartitionStatistics tableStatistics;
    if (table.getPartitionColumns().isEmpty()) {
        HiveBasicStatistics basicStatistics = partitionUpdates.stream().map(PartitionUpdate::getStatistics).reduce((first, second) -> reduce(first, second, ADD)).orElse(createZeroStatistics());
        tableStatistics = createPartitionStatistics(session, basicStatistics, columnTypes, getColumnStatistics(partitionComputedStatistics, ImmutableList.of()));
    } else {
        tableStatistics = new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of());
    }
    metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics);
    if (handle.getPartitionedBy().isEmpty()) {
        return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID)));
    }
    if (isRespectTableFormat(session)) {
        verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat());
    }
    for (PartitionUpdate update : partitionUpdates) {
        Map<String, String> partitionParameters = partitionEncryptionParameters;
        if (isPreferManifestsToListFiles(session) && isFileRenamingEnabled(session)) {
            // Store list of file names and sizes in partition metadata when prefer_manifests_to_list_files and file_renaming_enabled are set to true
            partitionParameters = updatePartitionMetadataWithFileNamesAndSizes(update, partitionParameters);
        }
        Partition partition = partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters);
        PartitionStatistics partitionStatistics = createPartitionStatistics(session, update.getStatistics(), columnTypes, getColumnStatistics(partitionComputedStatistics, partition.getValues()));
        metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), table.getStorage().getLocation(), true, partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters), update.getWritePath(), partitionStatistics);
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(toList())));
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Statistics.createEmptyPartitionStatistics(com.facebook.presto.hive.metastore.Statistics.createEmptyPartitionStatistics) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FILE_SIZE_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME) HiveSessionProperties.isPreferManifestsToListFiles(com.facebook.presto.hive.HiveSessionProperties.isPreferManifestsToListFiles) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) HiveSessionProperties.isOfflineDataDebugModeEnabled(com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) DWRF_ENCRYPTION_PROVIDER(com.facebook.presto.hive.HiveTableProperties.DWRF_ENCRYPTION_PROVIDER) MATERIALIZED_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW) HiveSessionProperties.isOptimizedPartitionUpdateSerializationEnabled(com.facebook.presto.hive.HiveSessionProperties.isOptimizedPartitionUpdateSerializationEnabled) Statistics.fromComputedStatistics(com.facebook.presto.hive.metastore.Statistics.fromComputedStatistics) MAX_VALUE_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE_SIZE_IN_BYTES) HiveTableProperties.getPartitionedBy(com.facebook.presto.hive.HiveTableProperties.getPartitionedBy) Map(java.util.Map) LocalProperty(com.facebook.presto.spi.LocalProperty) SystemTable(com.facebook.presto.spi.SystemTable) ENGLISH(java.util.Locale.ENGLISH) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) PATH_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME) HivePrivilegeInfo.toHivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) Collectors.joining(java.util.stream.Collectors.joining) Stream(java.util.stream.Stream) MIN_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MIN_VALUE) NUMBER_OF_DISTINCT_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_DISTINCT_VALUES) CSV_SEPARATOR(com.facebook.presto.hive.HiveTableProperties.CSV_SEPARATOR) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Joiner(com.google.common.base.Joiner) ConnectorPartitioningHandle(com.facebook.presto.spi.connector.ConnectorPartitioningHandle) Table(com.facebook.presto.hive.metastore.Table) Database(com.facebook.presto.hive.metastore.Database) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) HIVE_BINARY(com.facebook.presto.hive.HiveType.HIVE_BINARY) FULLY_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.FULLY_MATERIALIZED) HiveBucketHandle.createVirtualBucketHandle(com.facebook.presto.hive.HiveBucketHandle.createVirtualBucketHandle) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveTableProperties.getBucketProperty(com.facebook.presto.hive.HiveTableProperties.getBucketProperty) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) TableStatisticType(com.facebook.presto.spi.statistics.TableStatisticType) Supplier(java.util.function.Supplier) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) HiveMaterializedViewUtils.viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(com.facebook.presto.hive.HiveMaterializedViewUtils.viewToBaseTableOnOuterJoinSideIndirectMappedPartitions) OptionalLong(java.util.OptionalLong) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) Lists(com.google.common.collect.Lists) MaterializedDataPredicates(com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ENCRYPT_COLUMNS(com.facebook.presto.hive.HiveTableProperties.ENCRYPT_COLUMNS) Functions(com.google.common.base.Functions) HiveWriterFactory.computeBucketedFileName(com.facebook.presto.hive.HiveWriterFactory.computeBucketedFileName) RESPECT_TABLE_FORMAT(com.facebook.presto.hive.HiveSessionProperties.RESPECT_TABLE_FORMAT) IOException(java.io.IOException) Domain(com.facebook.presto.common.predicate.Domain) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) HiveUtil.hiveColumnHandles(com.facebook.presto.hive.HiveUtil.hiveColumnHandles) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) TableLayoutFilterCoverage(com.facebook.presto.spi.TableLayoutFilterCoverage) HiveManifestUtils.getManifestSizeInBytes(com.facebook.presto.hive.HiveManifestUtils.getManifestSizeInBytes) CSV_QUOTE(com.facebook.presto.hive.HiveTableProperties.CSV_QUOTE) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) HiveAnalyzeProperties.getPartitionList(com.facebook.presto.hive.HiveAnalyzeProperties.getPartitionList) NUMBER_OF_TRUE_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_TRUE_VALUES) JsonCodec(com.facebook.airlift.json.JsonCodec) DWRF_ENCRYPTION_ALGORITHM(com.facebook.presto.hive.HiveTableProperties.DWRF_ENCRYPTION_ALGORITHM) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveUtil.encodeMaterializedViewData(com.facebook.presto.hive.HiveUtil.encodeMaterializedViewData) StandardFunctionResolution(com.facebook.presto.spi.function.StandardFunctionResolution) Privilege(com.facebook.presto.spi.security.Privilege) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) HiveSessionProperties.getTemporaryTableSchema(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ADD(com.facebook.presto.hive.metastore.Statistics.ReduceOperator.ADD) ConnectorTablePartitioning(com.facebook.presto.spi.ConnectorTablePartitioning) Collectors.toMap(java.util.stream.Collectors.toMap) MaterializedViewNotFoundException(com.facebook.presto.spi.MaterializedViewNotFoundException) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) PRESTO_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) SpecialFormExpression(com.facebook.presto.spi.relation.SpecialFormExpression) ImmutableSet(com.google.common.collect.ImmutableSet) HIVE_UNKNOWN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR) INVALID_SCHEMA_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) SCHEMA_NOT_EMPTY(com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) DwrfTableEncryptionProperties.forPerColumn(com.facebook.presto.hive.DwrfTableEncryptionProperties.forPerColumn) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) LogicalRowExpressions.binaryExpression(com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression) TOTAL_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.TOTAL_SIZE_IN_BYTES) IntStream(java.util.stream.IntStream) OVERWRITE(com.facebook.presto.hive.PartitionUpdate.UpdateMode.OVERWRITE) MapType(com.facebook.presto.common.type.MapType) HiveSessionProperties.isRespectTableFormat(com.facebook.presto.hive.HiveSessionProperties.isRespectTableFormat) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) CompletableFuture(java.util.concurrent.CompletableFuture) HiveUtil.verifyPartitionTypeSupported(com.facebook.presto.hive.HiveUtil.verifyPartitionTypeSupported) OptionalInt(java.util.OptionalInt) Function(java.util.function.Function) InMemoryRecordSet(com.facebook.presto.spi.InMemoryRecordSet) HashSet(java.util.HashSet) UNPARTITIONED_ID(com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID) HiveMaterializedViewUtils.getViewToBasePartitionMap(com.facebook.presto.hive.HiveMaterializedViewUtils.getViewToBasePartitionMap) OpenCSVSerde(org.apache.hadoop.hive.serde2.OpenCSVSerde) PREFERRED_ORDERING_COLUMNS(com.facebook.presto.hive.HiveTableProperties.PREFERRED_ORDERING_COLUMNS) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) ALREADY_EXISTS(com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS) SmileCodec(com.facebook.airlift.json.smile.SmileCodec) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) NoSuchElementException(java.util.NoSuchElementException) HiveTableProperties.getDwrfEncryptionAlgorithm(com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionAlgorithm) Type(com.facebook.presto.common.type.Type) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) USER(com.facebook.presto.spi.security.PrincipalType.USER) Storage(com.facebook.presto.hive.metastore.Storage) HivePartitioningHandle.createHiveCompatiblePartitioningHandle(com.facebook.presto.hive.HivePartitioningHandle.createHiveCompatiblePartitioningHandle) ROW_COUNT(com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) HiveBucketing.getHiveBucketHandle(com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle) HiveTableProperties.getOrcBloomFilterColumns(com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterColumns) MoreFutures.toCompletableFuture(com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveMaterializedViewUtils.validateMaterializedViewPartitionColumns(com.facebook.presto.hive.HiveMaterializedViewUtils.validateMaterializedViewPartitionColumns) HiveTableProperties.getOrcBloomFilterFpp(com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterFpp) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) HiveSessionProperties.getTemporaryTableStorageFormat(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableStorageFormat) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) SYNTHESIZED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED) AVRO_SCHEMA_URL_KEY(com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY) Comparator(java.util.Comparator) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) NUMBER_OF_NON_NULL_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_NON_NULL_VALUES) HiveSessionProperties.isParquetPushdownFilterEnabled(com.facebook.presto.hive.HiveSessionProperties.isParquetPushdownFilterEnabled) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) HiveSessionProperties.isStatisticsEnabled(com.facebook.presto.hive.HiveSessionProperties.isStatisticsEnabled) EXTERNAL_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) AVRO_SCHEMA_URL(com.facebook.presto.hive.HiveTableProperties.AVRO_SCHEMA_URL) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) HiveSessionProperties.getBucketFunctionTypeForExchange(com.facebook.presto.hive.HiveSessionProperties.getBucketFunctionTypeForExchange) HiveMaterializedViewUtils.getMaterializedDataPredicates(com.facebook.presto.hive.HiveMaterializedViewUtils.getMaterializedDataPredicates) HiveTableProperties.getCsvProperty(com.facebook.presto.hive.HiveTableProperties.getCsvProperty) Predicates.not(com.google.common.base.Predicates.not) NOT_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.NOT_MATERIALIZED) ConnectorMaterializedViewDefinition(com.facebook.presto.spi.ConnectorMaterializedViewDefinition) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) FileWriteInfo(com.facebook.presto.hive.PartitionUpdate.FileWriteInfo) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) PrivilegeInfo(com.facebook.presto.spi.security.PrivilegeInfo) TEMPORARY_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) HiveUtil.decodeMaterializedViewData(com.facebook.presto.hive.HiveUtil.decodeMaterializedViewData) HiveManifestUtils.updatePartitionMetadataWithFileNamesAndSizes(com.facebook.presto.hive.HiveManifestUtils.updatePartitionMetadataWithFileNamesAndSizes) HIVE_INVALID_METADATA(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) HIVE_STORAGE_FORMAT(com.facebook.presto.hive.HiveSessionProperties.HIVE_STORAGE_FORMAT) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TypeUtils.isNumericType(com.facebook.presto.common.type.TypeUtils.isNumericType) JsonCodec.jsonCodec(com.facebook.airlift.json.JsonCodec.jsonCodec) ConnectorSession(com.facebook.presto.spi.ConnectorSession) CSV_ESCAPE(com.facebook.presto.hive.HiveTableProperties.CSV_ESCAPE) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) TableStatisticsMetadata(com.facebook.presto.spi.statistics.TableStatisticsMetadata) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) HiveColumnHandle.updateRowIdHandle(com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle) HiveSessionProperties.getTemporaryTableCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableCompressionCodec) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) HiveSessionProperties.isShufflePartitionedColumnsForTableWriteEnabled(com.facebook.presto.hive.HiveSessionProperties.isShufflePartitionedColumnsForTableWriteEnabled) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) HiveUtil.getPartitionKeyColumnHandles(com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles) HiveSessionProperties.isCollectColumnStatisticsOnWrite(com.facebook.presto.hive.HiveSessionProperties.isCollectColumnStatisticsOnWrite) ENCRYPT_TABLE(com.facebook.presto.hive.HiveTableProperties.ENCRYPT_TABLE) HiveTableProperties.getPreferredOrderingColumns(com.facebook.presto.hive.HiveTableProperties.getPreferredOrderingColumns) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HiveStatisticsProvider(com.facebook.presto.hive.statistics.HiveStatisticsProvider) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) Builder(com.google.common.collect.ImmutableMap.Builder) ArrayList(java.util.ArrayList) SortingProperty(com.facebook.presto.spi.SortingProperty) HiveTableProperties.getDwrfEncryptionProvider(com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionProvider) DwrfTableEncryptionProperties.fromHiveTableProperties(com.facebook.presto.hive.DwrfTableEncryptionProperties.fromHiveTableProperties) HiveUtil.encodeViewData(com.facebook.presto.hive.HiveUtil.encodeViewData) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) HiveBucketFilter(com.facebook.presto.hive.HiveBucketing.HiveBucketFilter) HiveWriteUtils.checkTableIsWritable(com.facebook.presto.hive.HiveWriteUtils.checkTableIsWritable) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) ADMIN_ROLE_NAME(com.facebook.presto.hive.security.SqlStandardAccessControl.ADMIN_ROLE_NAME) ConnectorMetadataUpdateHandle(com.facebook.presto.spi.ConnectorMetadataUpdateHandle) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) COVERED(com.facebook.presto.spi.TableLayoutFilterCoverage.COVERED) HiveSessionProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat) HiveSessionProperties.isOptimizedMismatchedBucketCount(com.facebook.presto.hive.HiveSessionProperties.isOptimizedMismatchedBucketCount) Properties(java.util.Properties) HiveSessionProperties.getCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getCompressionCodec) HIVE_CONCURRENT_MODIFICATION_DETECTED(com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED) Constraint(com.facebook.presto.spi.Constraint) File(java.io.File) PRESTO_MATERIALIZED_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG) Streams.stream(com.google.common.collect.Streams.stream) ColumnWithStructSubfield(com.facebook.presto.hive.ColumnEncryptionInformation.ColumnWithStructSubfield) INVALID_VIEW(com.facebook.presto.spi.StandardErrorCode.INVALID_VIEW) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HiveTableProperties.getEncryptColumns(com.facebook.presto.hive.HiveTableProperties.getEncryptColumns) INVALID_ANALYZE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY) QueryId(com.facebook.presto.spi.QueryId) HiveTableProperties.getAvroSchemaUrl(com.facebook.presto.hive.HiveTableProperties.getAvroSchemaUrl) HiveMaterializedViewUtils.differenceDataPredicates(com.facebook.presto.hive.HiveMaterializedViewUtils.differenceDataPredicates) URL(java.net.URL) HiveTableProperties.getExternalLocation(com.facebook.presto.hive.HiveTableProperties.getExternalLocation) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) HiveSessionProperties.isUsePageFileForHiveUnsupportedType(com.facebook.presto.hive.HiveSessionProperties.isUsePageFileForHiveUnsupportedType) FILE_MODIFIED_TIME_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME) HiveSessionProperties.isBucketExecutionEnabled(com.facebook.presto.hive.HiveSessionProperties.isBucketExecutionEnabled) Iterables.concat(com.google.common.collect.Iterables.concat) HIVE_COLUMN_ORDER_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) Splitter(com.google.common.base.Splitter) Collectors.toSet(java.util.stream.Collectors.toSet) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) HiveSessionProperties.isSortedWriteToTempPathEnabled(com.facebook.presto.hive.HiveSessionProperties.isSortedWriteToTempPathEnabled) ORC_BLOOM_FILTER_FPP(com.facebook.presto.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP) ImmutableMap(com.google.common.collect.ImmutableMap) HivePartitioningHandle.createPrestoNativePartitioningHandle(com.facebook.presto.hive.HivePartitioningHandle.createPrestoNativePartitioningHandle) NEW(com.facebook.presto.hive.PartitionUpdate.UpdateMode.NEW) NOT_COVERED(com.facebook.presto.spi.TableLayoutFilterCoverage.NOT_COVERED) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) MAX_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE) HiveSessionProperties.isStreamingAggregationEnabled(com.facebook.presto.hive.HiveSessionProperties.isStreamingAggregationEnabled) Sets(com.google.common.collect.Sets) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) RoleGrant(com.facebook.presto.spi.security.RoleGrant) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) HIVE_TIMEZONE_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_TIMEZONE_MISMATCH) ORC_BLOOM_FILTER_COLUMNS(com.facebook.presto.hive.HiveTableProperties.ORC_BLOOM_FILTER_COLUMNS) TOO_MANY_PARTITIONS_MISSING(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.TOO_MANY_PARTITIONS_MISSING) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) PRESTO_NATIVE(com.facebook.presto.hive.BucketFunctionType.PRESTO_NATIVE) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) FilterStatsCalculatorService(com.facebook.presto.spi.plan.FilterStatsCalculatorService) HiveWriterFactory.getFileExtension(com.facebook.presto.hive.HiveWriterFactory.getFileExtension) MetastoreUtil.getProtectMode(com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode) HiveSessionProperties.isSortedWritingEnabled(com.facebook.presto.hive.HiveSessionProperties.isSortedWritingEnabled) Column(com.facebook.presto.hive.metastore.Column) ColumnStatisticMetadata(com.facebook.presto.spi.statistics.ColumnStatisticMetadata) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) PrestoException(com.facebook.presto.spi.PrestoException) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Verify.verify(com.google.common.base.Verify.verify) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getVirtualBucketCount(com.facebook.presto.hive.HiveSessionProperties.getVirtualBucketCount) Objects.requireNonNull(java.util.Objects.requireNonNull) MetastoreUtil.verifyOnline(com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline) Suppliers(com.google.common.base.Suppliers) HiveSessionProperties.isFileRenamingEnabled(com.facebook.presto.hive.HiveSessionProperties.isFileRenamingEnabled) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) VerifyException(com.google.common.base.VerifyException) GrantInfo(com.facebook.presto.spi.security.GrantInfo) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) NOT_APPLICABLE(com.facebook.presto.spi.TableLayoutFilterCoverage.NOT_APPLICABLE) MalformedURLException(java.net.MalformedURLException) HIVE_UNSUPPORTED_ENCRYPTION_OPERATION(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_ENCRYPTION_OPERATION) Statistics.reduce(com.facebook.presto.hive.metastore.Statistics.reduce) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) EXTERNAL_LOCATION_PROPERTY(com.facebook.presto.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY) HiveSessionProperties.isCreateEmptyBucketFiles(com.facebook.presto.hive.HiveSessionProperties.isCreateEmptyBucketFiles) VIEW_STORAGE_FORMAT(com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) HiveUtil.deserializeZstdCompressed(com.facebook.presto.hive.HiveUtil.deserializeZstdCompressed) Maps(com.google.common.collect.Maps) ThriftMetastoreUtil.listEnabledPrincipals(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.listEnabledPrincipals) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) VIRTUAL_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) PARTIALLY_MATERIALIZED(com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.PARTIALLY_MATERIALIZED) UUID.randomUUID(java.util.UUID.randomUUID) ThriftMetastoreUtil(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) HIVE_PARTITION_READ_ONLY(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_READ_ONLY) HiveStorageFormat.values(com.facebook.presto.hive.HiveStorageFormat.values) APPEND(com.facebook.presto.hive.PartitionUpdate.UpdateMode.APPEND) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) MetastoreUtil.isUserDefinedTypeEncodingEnabled(com.facebook.presto.hive.metastore.MetastoreUtil.isUserDefinedTypeEncodingEnabled) HiveUtil.decodeViewData(com.facebook.presto.hive.HiveUtil.decodeViewData) MaterializedViewStatus(com.facebook.presto.spi.MaterializedViewStatus) VisibleForTesting(com.google.common.annotations.VisibleForTesting) HiveSessionProperties.getOrcCompressionCodec(com.facebook.presto.hive.HiveSessionProperties.getOrcCompressionCodec) Block(com.facebook.presto.common.block.Block) Statistics.createComputedStatisticsToPartitionMap(com.facebook.presto.hive.metastore.Statistics.createComputedStatisticsToPartitionMap) Partition(com.facebook.presto.hive.metastore.Partition) SystemTable(com.facebook.presto.spi.SystemTable) DwrfTableEncryptionProperties.forTable(com.facebook.presto.hive.DwrfTableEncryptionProperties.forTable) HiveUtil.translateHiveUnsupportedTypeForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable) Table(com.facebook.presto.hive.metastore.Table) HiveUtil.translateHiveUnsupportedTypesForTemporaryTable(com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable) HiveTableProperties.getEncryptTable(com.facebook.presto.hive.HiveTableProperties.getEncryptTable) HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable(com.facebook.presto.hive.HiveSessionProperties.shouldCreateEmptyBucketFilesForTemporaryTable) HiveTableProperties.isExternalTable(com.facebook.presto.hive.HiveTableProperties.isExternalTable) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) TableStatisticType(com.facebook.presto.spi.statistics.TableStatisticType) RowType(com.facebook.presto.common.type.RowType) MapType(com.facebook.presto.common.type.MapType) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) Type(com.facebook.presto.common.type.Type) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) TypeUtils.isNumericType(com.facebook.presto.common.type.TypeUtils.isNumericType) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) ArrayType(com.facebook.presto.common.type.ArrayType) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) HiveSessionProperties.isUsePageFileForHiveUnsupportedType(com.facebook.presto.hive.HiveSessionProperties.isUsePageFileForHiveUnsupportedType) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) Statistics.createEmptyPartitionStatistics(com.facebook.presto.hive.metastore.Statistics.createEmptyPartitionStatistics) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) Statistics.fromComputedStatistics(com.facebook.presto.hive.metastore.Statistics.fromComputedStatistics) FileWriteInfo(com.facebook.presto.hive.PartitionUpdate.FileWriteInfo) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HiveAnalyzeProperties.getPartitionList(com.facebook.presto.hive.HiveAnalyzeProperties.getPartitionList) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ArrayList(java.util.ArrayList) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List)

Example 60 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveFilterPushdown method pushdownFilter.

@VisibleForTesting
public static ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, ConnectorMetadata metadata, SemiTransactionalHiveMetastore metastore, RowExpressionService rowExpressionService, StandardFunctionResolution functionResolution, HivePartitionManager partitionManager, FunctionMetadataManager functionMetadataManager, ConnectorTableHandle tableHandle, RowExpression filter, Optional<ConnectorTableLayoutHandle> currentLayoutHandle) {
    checkArgument(!FALSE_CONSTANT.equals(filter), "Cannot pushdown filter that is always false");
    if (TRUE_CONSTANT.equals(filter) && currentLayoutHandle.isPresent()) {
        return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT);
    }
    // Split the filter into 3 groups of conjuncts:
    // - range filters that apply to entire columns,
    // - range filters that apply to subfields,
    // - the rest. Intersect these with possibly pre-existing filters.
    DomainTranslator.ExtractionResult<Subfield> decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor());
    if (currentLayoutHandle.isPresent()) {
        HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get();
        decomposedFilter = intersectExtractionResult(new DomainTranslator.ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate()), decomposedFilter);
    }
    if (decomposedFilter.getTupleDomain().isNone()) {
        return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
    }
    RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer().optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session);
    if (optimizedRemainingExpression instanceof ConstantExpression) {
        ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression;
        if (FALSE_CONSTANT.equals(constantExpression) || constantExpression.getValue() == null) {
            return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
        }
    }
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    TupleDomain<ColumnHandle> entireColumnDomain = decomposedFilter.getTupleDomain().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(columnHandles::get);
    if (currentLayoutHandle.isPresent()) {
        entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate());
    }
    Constraint<ColumnHandle> constraint = new Constraint<>(entireColumnDomain);
    // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate
    if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) {
        LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
        RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression());
        if (!TRUE_CONSTANT.equals(deterministicPredicate)) {
            ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate);
            constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate);
        }
    }
    HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    TupleDomain<Subfield> domainPredicate = withColumnDomains(ImmutableMap.<Subfield, Domain>builder().putAll(hivePartitionResult.getUnenforcedConstraint().transform(HiveFilterPushdown::toSubfield).getDomains().orElse(ImmutableMap.of())).putAll(decomposedFilter.getTupleDomain().transform(subfield -> !isEntireColumn(subfield) ? subfield : null).getDomains().orElse(ImmutableMap.of())).build());
    Set<String> predicateColumnNames = new HashSet<>();
    domainPredicate.getDomains().get().keySet().stream().map(Subfield::getRootName).forEach(predicateColumnNames::add);
    // Include only columns referenced in the optimized expression. Although the expression is sent to the worker node
    // unoptimized, the worker is expected to optimize the expression before executing.
    extractAll(optimizedRemainingExpression).stream().map(VariableReferenceExpression::getName).forEach(predicateColumnNames::add);
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
    List<RowExpression> conjuncts = extractConjuncts(decomposedFilter.getRemainingExpression());
    RowExpression dynamicFilterExpression = extractDynamicConjuncts(conjuncts, logicalRowExpressions);
    RowExpression remainingExpression = extractStaticConjuncts(conjuncts, logicalRowExpressions);
    remainingExpression = removeNestedDynamicFilters(remainingExpression);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, new HiveTableLayoutHandle(tableName, table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, remainingExpression, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), true, createTableLayoutString(session, rowExpressionService, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), remainingExpression, domainPredicate), currentLayoutHandle.map(layout -> ((HiveTableLayoutHandle) layout).getRequestedColumns()).orElse(Optional.empty()), false)), dynamicFilterExpression);
}
Also used : LogicalRowExpressions.extractConjuncts(com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts) HiveSessionProperties.isParquetPushdownFilterEnabled(com.facebook.presto.hive.HiveSessionProperties.isParquetPushdownFilterEnabled) HiveTableLayoutHandle(com.facebook.presto.hive.HiveTableLayoutHandle) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) PlanVisitor(com.facebook.presto.spi.plan.PlanVisitor) StandardFunctionResolution(com.facebook.presto.spi.function.StandardFunctionResolution) HiveTransactionManager(com.facebook.presto.hive.HiveTransactionManager) ValuesNode(com.facebook.presto.spi.plan.ValuesNode) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Map(java.util.Map) FALSE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT) INVALID_FUNCTION_ARGUMENT(com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT) HiveBucketHandle(com.facebook.presto.hive.HiveBucketHandle) BiMap(com.google.common.collect.BiMap) ImmutableSet(com.google.common.collect.ImmutableSet) NullableValue(com.facebook.presto.common.predicate.NullableValue) ImmutableMap(com.google.common.collect.ImmutableMap) Collections.emptyList(java.util.Collections.emptyList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) DomainTranslator(com.facebook.presto.spi.relation.DomainTranslator) Set(java.util.Set) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ConnectorSession(com.facebook.presto.spi.ConnectorSession) DynamicFilters.extractDynamicConjuncts(com.facebook.presto.expressions.DynamicFilters.extractDynamicConjuncts) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) DefaultRowExpressionTraversalVisitor(com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor) HiveTableHandle(com.facebook.presto.hive.HiveTableHandle) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) HivePartitionResult(com.facebook.presto.hive.HivePartitionResult) Optional(java.util.Optional) HiveColumnHandle(com.facebook.presto.hive.HiveColumnHandle) HiveSessionProperties(com.facebook.presto.hive.HiveSessionProperties) DynamicFilters.removeNestedDynamicFilters(com.facebook.presto.expressions.DynamicFilters.removeNestedDynamicFilters) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorPlanOptimizer(com.facebook.presto.spi.ConnectorPlanOptimizer) Table(com.facebook.presto.hive.metastore.Table) DIVISION_BY_ZERO(com.facebook.presto.spi.StandardErrorCode.DIVISION_BY_ZERO) Column(com.facebook.presto.hive.metastore.Column) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) ConstantExpression(com.facebook.presto.spi.relation.ConstantExpression) Function(java.util.function.Function) HashSet(java.util.HashSet) FilterNode(com.facebook.presto.spi.plan.FilterNode) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) LogicalRowExpressions(com.facebook.presto.expressions.LogicalRowExpressions) VariableAllocator(com.facebook.presto.spi.VariableAllocator) Objects.requireNonNull(java.util.Objects.requireNonNull) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) TableHandle(com.facebook.presto.spi.TableHandle) FunctionMetadataManager(com.facebook.presto.spi.function.FunctionMetadataManager) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) HivePartitionManager(com.facebook.presto.hive.HivePartitionManager) RowExpression(com.facebook.presto.spi.relation.RowExpression) Functions(com.google.common.base.Functions) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) SubfieldExtractor(com.facebook.presto.hive.SubfieldExtractor) LogicalRowExpressions.and(com.facebook.presto.expressions.LogicalRowExpressions.and) INVALID_CAST_ARGUMENT(com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT) Constraint(com.facebook.presto.spi.Constraint) HiveBucketing(com.facebook.presto.hive.HiveBucketing) Domain(com.facebook.presto.common.predicate.Domain) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) OPTIMIZED(com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED) HiveMetadata(com.facebook.presto.hive.HiveMetadata) PlanNode(com.facebook.presto.spi.plan.PlanNode) DynamicFilters.extractStaticConjuncts(com.facebook.presto.expressions.DynamicFilters.extractStaticConjuncts) NUMERIC_VALUE_OUT_OF_RANGE(com.facebook.presto.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HiveStorageFormat(com.facebook.presto.hive.HiveStorageFormat) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) Sets.intersection(com.google.common.collect.Sets.intersection) MetastoreUtil.isUserDefinedTypeEncodingEnabled(com.facebook.presto.hive.metastore.MetastoreUtil.isUserDefinedTypeEncodingEnabled) ImmutableBiMap.toImmutableBiMap(com.google.common.collect.ImmutableBiMap.toImmutableBiMap) VisibleForTesting(com.google.common.annotations.VisibleForTesting) RowExpressionNodeInliner.replaceExpression(com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression) HivePartitionResult(com.facebook.presto.hive.HivePartitionResult) Constraint(com.facebook.presto.spi.Constraint) ConstantExpression(com.facebook.presto.spi.relation.ConstantExpression) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) SubfieldExtractor(com.facebook.presto.hive.SubfieldExtractor) HiveTableHandle(com.facebook.presto.hive.HiveTableHandle) DomainTranslator(com.facebook.presto.spi.relation.DomainTranslator) HiveTableLayoutHandle(com.facebook.presto.hive.HiveTableLayoutHandle) Subfield(com.facebook.presto.common.Subfield) HiveColumnHandle(com.facebook.presto.hive.HiveColumnHandle) HashSet(java.util.HashSet) HiveColumnHandle(com.facebook.presto.hive.HiveColumnHandle) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Table(com.facebook.presto.hive.metastore.Table) LogicalRowExpressions(com.facebook.presto.expressions.LogicalRowExpressions) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) RowExpression(com.facebook.presto.spi.relation.RowExpression) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Domain(com.facebook.presto.common.predicate.Domain) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

MetastoreContext (com.facebook.presto.hive.metastore.MetastoreContext)99 Table (com.facebook.presto.hive.metastore.Table)59 PrestoException (com.facebook.presto.spi.PrestoException)53 SchemaTableName (com.facebook.presto.spi.SchemaTableName)52 TableNotFoundException (com.facebook.presto.spi.TableNotFoundException)49 Column (com.facebook.presto.hive.metastore.Column)42 ImmutableList (com.google.common.collect.ImmutableList)41 ImmutableMap (com.google.common.collect.ImmutableMap)41 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)40 Map (java.util.Map)38 Optional (java.util.Optional)38 PrestoPrincipal (com.facebook.presto.spi.security.PrestoPrincipal)37 List (java.util.List)37 Set (java.util.Set)37 PartitionStatistics (com.facebook.presto.hive.metastore.PartitionStatistics)35 Objects.requireNonNull (java.util.Objects.requireNonNull)35 Type (com.facebook.presto.common.type.Type)34 HivePrivilegeInfo (com.facebook.presto.hive.metastore.HivePrivilegeInfo)34 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)34 Domain (com.facebook.presto.common.predicate.Domain)33