Search in sources :

Example 1 with TABLE_COMMENT

use of io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT in project trino by trinodb.

the class BridgingHiveMetastore method commentTable.

@Override
public void commentTable(String databaseName, String tableName, Optional<String> comment) {
    Optional<org.apache.hadoop.hive.metastore.api.Table> source = delegate.getTable(identity, databaseName, tableName);
    if (source.isEmpty()) {
        throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
    }
    org.apache.hadoop.hive.metastore.api.Table table = source.get();
    Map<String, String> parameters = table.getParameters().entrySet().stream().filter(entry -> !entry.getKey().equals(TABLE_COMMENT)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    comment.ifPresent(value -> parameters.put(TABLE_COMMENT, value));
    table.setParameters(parameters);
    alterTable(databaseName, tableName, table);
}
Also used : ThriftMetastoreUtil.fromMetastoreApiDatabase(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiDatabase) UnaryOperator.identity(java.util.function.UnaryOperator.identity) USER(io.trino.spi.security.PrincipalType.USER) ThriftMetastoreUtil.toMetastoreApiDatabase(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiDatabase) Database(io.trino.plugin.hive.metastore.Database) AcidOperation(io.trino.plugin.hive.acid.AcidOperation) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) ThriftMetastoreUtil.csvSchemaFields(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.csvSchemaFields) AcidTransactionOwner(io.trino.plugin.hive.metastore.AcidTransactionOwner) ColumnStatisticType(io.trino.spi.statistics.ColumnStatisticType) ThriftMetastoreUtil.fromMetastoreApiTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Map(java.util.Map) PartitionWithStatistics(io.trino.plugin.hive.metastore.PartitionWithStatistics) TABLE_COMMENT(io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT) HiveIdentity(io.trino.plugin.hive.authentication.HiveIdentity) AcidTransaction(io.trino.plugin.hive.acid.AcidTransaction) Table(io.trino.plugin.hive.metastore.Table) ImmutableMap(com.google.common.collect.ImmutableMap) HivePartition(io.trino.plugin.hive.HivePartition) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TrinoException(io.trino.spi.TrinoException) Collectors(java.util.stream.Collectors) SchemaTableName(io.trino.spi.connector.SchemaTableName) ThriftMetastoreUtil.toMetastoreApiTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiTable) List(java.util.List) ThriftMetastoreUtil.isCsvTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.isCsvTable) Optional(java.util.Optional) HivePrivilegeInfo(io.trino.plugin.hive.metastore.HivePrivilegeInfo) Partition(io.trino.plugin.hive.metastore.Partition) PartitionStatistics(io.trino.plugin.hive.PartitionStatistics) HivePrincipal(io.trino.plugin.hive.metastore.HivePrincipal) HiveUtil(io.trino.plugin.hive.util.HiveUtil) MetastoreUtil.isAvroTableWithSchemaSet(io.trino.plugin.hive.metastore.MetastoreUtil.isAvroTableWithSchemaSet) Type(io.trino.spi.type.Type) ThriftMetastoreUtil.isAvroTableWithSchemaSet(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.isAvroTableWithSchemaSet) Function(java.util.function.Function) DataOperationType(org.apache.hadoop.hive.metastore.api.DataOperationType) HiveType(io.trino.plugin.hive.HiveType) OptionalLong(java.util.OptionalLong) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) Objects.requireNonNull(java.util.Objects.requireNonNull) TupleDomain(io.trino.spi.predicate.TupleDomain) RoleGrant(io.trino.spi.security.RoleGrant) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) MetastoreUtil.verifyCanDropColumn(io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) HivePrivilege(io.trino.plugin.hive.metastore.HivePrivilegeInfo.HivePrivilege) ThriftMetastoreUtil.fromMetastoreApiTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable) Table(io.trino.plugin.hive.metastore.Table) ThriftMetastoreUtil.toMetastoreApiTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiTable) ThriftMetastoreUtil.isCsvTable(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.isCsvTable) SchemaTableName(io.trino.spi.connector.SchemaTableName) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 2 with TABLE_COMMENT

use of io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT in project trino by trinodb.

the class AbstractMetastoreTableOperations method commitNewTable.

@Override
protected final void commitNewTable(TableMetadata metadata) {
    String newMetadataLocation = writeNewMetadata(metadata, version + 1);
    Table.Builder builder = Table.builder().setDatabaseName(database).setTableName(tableName).setOwner(owner).setTableType(TableType.EXTERNAL_TABLE.name()).setDataColumns(toHiveColumns(metadata.schema().columns())).withStorage(storage -> storage.setLocation(metadata.location())).withStorage(storage -> storage.setStorageFormat(STORAGE_FORMAT)).setParameter("EXTERNAL", "TRUE").setParameter(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE).setParameter(METADATA_LOCATION_PROP, newMetadataLocation);
    String tableComment = metadata.properties().get(TABLE_COMMENT);
    if (tableComment != null) {
        builder.setParameter(TABLE_COMMENT, tableComment);
    }
    Table table = builder.build();
    PrincipalPrivileges privileges = owner.map(MetastoreUtil::buildInitialPrivilegeSet).orElse(NO_PRIVILEGES);
    metastore.createTable(table, privileges);
}
Also used : UnknownTableTypeException(io.trino.plugin.iceberg.UnknownTableTypeException) AbstractIcebergTableOperations(io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations) ICEBERG_TABLE_TYPE_VALUE(org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE) TABLE_TYPE_PROP(org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP) TableMetadata(org.apache.iceberg.TableMetadata) IcebergUtil.isIcebergTable(io.trino.plugin.iceberg.IcebergUtil.isIcebergTable) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Objects.requireNonNull(java.util.Objects.requireNonNull) TABLE_COMMENT(io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT) ViewReaderUtil.isPrestoView(io.trino.plugin.hive.ViewReaderUtil.isPrestoView) Table(io.trino.plugin.hive.metastore.Table) NO_PRIVILEGES(io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES) TrinoException(io.trino.spi.TrinoException) ConnectorSession(io.trino.spi.connector.ConnectorSession) String.format(java.lang.String.format) ICEBERG_INVALID_METADATA(io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA) TableType(org.apache.hadoop.hive.metastore.TableType) Optional(java.util.Optional) METADATA_LOCATION_PROP(org.apache.iceberg.BaseMetastoreTableOperations.METADATA_LOCATION_PROP) ViewReaderUtil.isHiveOrPrestoView(io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) FileIO(org.apache.iceberg.io.FileIO) MetastoreUtil(io.trino.plugin.hive.metastore.MetastoreUtil) NotThreadSafe(javax.annotation.concurrent.NotThreadSafe) IcebergUtil.isIcebergTable(io.trino.plugin.iceberg.IcebergUtil.isIcebergTable) Table(io.trino.plugin.hive.metastore.Table) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges)

Example 3 with TABLE_COMMENT

use of io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT in project trino by trinodb.

the class TrinoHiveCatalog method createMaterializedView.

@Override
public void createMaterializedView(ConnectorSession session, SchemaTableName schemaViewName, ConnectorMaterializedViewDefinition definition, boolean replace, boolean ignoreExisting) {
    Optional<io.trino.plugin.hive.metastore.Table> existing = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
    // It's a create command where the materialized view already exists and 'if not exists' clause is not specified
    if (!replace && existing.isPresent()) {
        if (ignoreExisting) {
            return;
        }
        throw new TrinoException(ALREADY_EXISTS, "Materialized view already exists: " + schemaViewName);
    }
    // Generate a storage table name and create a storage table. The properties in the definition are table properties for the
    // storage table as indicated in the materialized view definition.
    String storageTableName = "st_" + randomUUID().toString().replace("-", "");
    Map<String, Object> storageTableProperties = new HashMap<>(definition.getProperties());
    storageTableProperties.putIfAbsent(FILE_FORMAT_PROPERTY, DEFAULT_FILE_FORMAT_DEFAULT);
    SchemaTableName storageTable = new SchemaTableName(schemaViewName.getSchemaName(), storageTableName);
    List<ColumnMetadata> columns = definition.getColumns().stream().map(column -> new ColumnMetadata(column.getName(), typeManager.getType(column.getType()))).collect(toImmutableList());
    ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(storageTable, columns, storageTableProperties, Optional.empty());
    Transaction transaction = IcebergUtil.newCreateTableTransaction(this, tableMetadata, session);
    transaction.newAppend().commit();
    transaction.commitTransaction();
    // Create a view indicating the storage table
    Map<String, String> viewProperties = ImmutableMap.<String, String>builder().put(PRESTO_QUERY_ID_NAME, session.getQueryId()).put(STORAGE_TABLE, storageTableName).put(PRESTO_VIEW_FLAG, "true").put(TRINO_CREATED_BY, TRINO_CREATED_BY_VALUE).put(TABLE_COMMENT, ICEBERG_MATERIALIZED_VIEW_COMMENT).buildOrThrow();
    Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty());
    io.trino.plugin.hive.metastore.Table.Builder tableBuilder = io.trino.plugin.hive.metastore.Table.builder().setDatabaseName(schemaViewName.getSchemaName()).setTableName(schemaViewName.getTableName()).setOwner(isUsingSystemSecurity ? Optional.empty() : Optional.of(session.getUser())).setTableType(VIRTUAL_VIEW.name()).setDataColumns(ImmutableList.of(dummyColumn)).setPartitionColumns(ImmutableList.of()).setParameters(viewProperties).withStorage(storage -> storage.setStorageFormat(VIEW_STORAGE_FORMAT)).withStorage(storage -> storage.setLocation("")).setViewOriginalText(Optional.of(encodeMaterializedViewData(fromConnectorMaterializedViewDefinition(definition)))).setViewExpandedText(Optional.of("/* Presto Materialized View */"));
    io.trino.plugin.hive.metastore.Table table = tableBuilder.build();
    PrincipalPrivileges principalPrivileges = isUsingSystemSecurity ? NO_PRIVILEGES : buildInitialPrivilegeSet(session.getUser());
    if (existing.isPresent() && replace) {
        // drop the current storage table
        String oldStorageTable = existing.get().getParameters().get(STORAGE_TABLE);
        if (oldStorageTable != null) {
            metastore.dropTable(schemaViewName.getSchemaName(), oldStorageTable, true);
        }
        // Replace the existing view definition
        metastore.replaceTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), table, principalPrivileges);
        return;
    }
    // create the view definition
    metastore.createTable(table, principalPrivileges);
}
Also used : FileSystem(org.apache.hadoop.fs.FileSystem) Throwables.throwIfUnchecked(com.google.common.base.Throwables.throwIfUnchecked) ICEBERG_TABLE_TYPE_VALUE(org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE) TABLE_TYPE_PROP(org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Column(io.trino.plugin.hive.metastore.Column) TABLE_NOT_FOUND(io.trino.spi.StandardErrorCode.TABLE_NOT_FOUND) Duration(java.time.Duration) Map(java.util.Map) ViewNotFoundException(io.trino.spi.connector.ViewNotFoundException) TABLE_COMMENT(io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) BaseTable(org.apache.iceberg.BaseTable) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Schema(org.apache.iceberg.Schema) IcebergUtil.validateTableCanBeDropped(io.trino.plugin.iceberg.IcebergUtil.validateTableCanBeDropped) SchemaTableName(io.trino.spi.connector.SchemaTableName) HiveMetadata(io.trino.plugin.hive.HiveMetadata) Stream(java.util.stream.Stream) TrinoPrincipal(io.trino.spi.security.TrinoPrincipal) STORAGE_TABLE(io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) CatalogUtil.dropTableData(org.apache.iceberg.CatalogUtil.dropTableData) HivePrincipal(io.trino.plugin.hive.metastore.HivePrincipal) HiveUtil(io.trino.plugin.hive.util.HiveUtil) HiveViewNotSupportedException(io.trino.plugin.hive.HiveViewNotSupportedException) IcebergMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) HIVE_INVALID_METADATA(io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) SCHEMA_NOT_EMPTY(io.trino.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) FILE_FORMAT_PROPERTY(io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY) AbstractTrinoCatalog(io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog) ViewReaderUtil.isPrestoView(io.trino.plugin.hive.ViewReaderUtil.isPrestoView) Table(org.apache.iceberg.Table) IOException(java.io.IOException) ConnectorSession(io.trino.spi.connector.ConnectorSession) CatalogName(io.trino.plugin.base.CatalogName) Failsafe(net.jodah.failsafe.Failsafe) HiveUtil.isHiveSystemSchema(io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema) ChronoUnit(java.time.temporal.ChronoUnit) ViewReaderUtil.isHiveOrPrestoView(io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView) MaterializedViewNotFoundException(io.trino.spi.connector.MaterializedViewNotFoundException) IcebergSchemaProperties.getSchemaLocation(io.trino.plugin.iceberg.IcebergSchemaProperties.getSchemaLocation) IcebergMaterializedViewDefinition.decodeMaterializedViewData(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.decodeMaterializedViewData) Database(io.trino.plugin.hive.metastore.Database) HiveSchemaProperties(io.trino.plugin.hive.HiveSchemaProperties) ConnectorMaterializedViewDefinition(io.trino.spi.connector.ConnectorMaterializedViewDefinition) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) TableMetadata(org.apache.iceberg.TableMetadata) IcebergUtil(io.trino.plugin.iceberg.IcebergUtil) Locale(java.util.Locale) ALREADY_EXISTS(io.trino.spi.StandardErrorCode.ALREADY_EXISTS) Path(org.apache.hadoop.fs.Path) ViewReaderUtil.encodeViewData(io.trino.plugin.hive.ViewReaderUtil.encodeViewData) ConnectorViewDefinition(io.trino.spi.connector.ConnectorViewDefinition) ImmutableSet(com.google.common.collect.ImmutableSet) HiveWriteUtils.getTableDefaultLocation(io.trino.plugin.hive.util.HiveWriteUtils.getTableDefaultLocation) ImmutableMap(com.google.common.collect.ImmutableMap) IcebergTableOperationsProvider(io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) VIEW_STORAGE_FORMAT(io.trino.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) TrinoException(io.trino.spi.TrinoException) RetryPolicy(net.jodah.failsafe.RetryPolicy) Collectors(java.util.stream.Collectors) Preconditions.checkState(com.google.common.base.Preconditions.checkState) PRESTO_VIEW_FLAG(io.trino.plugin.hive.ViewReaderUtil.PRESTO_VIEW_FLAG) INVALID_SCHEMA_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) List(java.util.List) IcebergUtil.loadIcebergTable(io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable) MetastoreUtil.buildInitialPrivilegeSet(io.trino.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet) PartitionSpec(org.apache.iceberg.PartitionSpec) Optional(java.util.Optional) Logger(io.airlift.log.Logger) PartitionFields.toPartitionFields(io.trino.plugin.iceberg.PartitionFields.toPartitionFields) ColumnIdentity(io.trino.plugin.iceberg.ColumnIdentity) ViewReaderUtil(io.trino.plugin.hive.ViewReaderUtil) HashMap(java.util.HashMap) IcebergUtil.getIcebergTableWithMetadata(io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata) PARTITIONING_PROPERTY(io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) CachingHiveMetastore(io.trino.plugin.hive.metastore.cache.CachingHiveMetastore) VIRTUAL_VIEW(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW) IcebergMaterializedViewDefinition.encodeMaterializedViewData(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.encodeMaterializedViewData) NO_PRIVILEGES(io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES) ViewAlreadyExistsException(io.trino.plugin.hive.ViewAlreadyExistsException) HIVE_STRING(io.trino.plugin.hive.HiveType.HIVE_STRING) IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition) UUID.randomUUID(java.util.UUID.randomUUID) Transaction(org.apache.iceberg.Transaction) DEFAULT_FILE_FORMAT_DEFAULT(org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) TypeManager(io.trino.spi.type.TypeManager) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) BaseTable(org.apache.iceberg.BaseTable) Table(org.apache.iceberg.Table) IcebergUtil.loadIcebergTable(io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) SchemaTableName(io.trino.spi.connector.SchemaTableName) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) Transaction(org.apache.iceberg.Transaction) Column(io.trino.plugin.hive.metastore.Column) TrinoException(io.trino.spi.TrinoException) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata)

Aggregations

TABLE_COMMENT (io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT)3 HiveMetastore (io.trino.plugin.hive.metastore.HiveMetastore)3 PrincipalPrivileges (io.trino.plugin.hive.metastore.PrincipalPrivileges)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ViewReaderUtil.isHiveOrPrestoView (io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView)2 ViewReaderUtil.isPrestoView (io.trino.plugin.hive.ViewReaderUtil.isPrestoView)2 Database (io.trino.plugin.hive.metastore.Database)2 HivePrincipal (io.trino.plugin.hive.metastore.HivePrincipal)2 NO_PRIVILEGES (io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES)2 Table (io.trino.plugin.hive.metastore.Table)2 HiveUtil (io.trino.plugin.hive.util.HiveUtil)2 TrinoException (io.trino.spi.TrinoException)2 TableNotFoundException (io.trino.spi.connector.TableNotFoundException)2 Objects.requireNonNull (java.util.Objects.requireNonNull)2 Optional (java.util.Optional)2 Preconditions.checkState (com.google.common.base.Preconditions.checkState)1 Throwables.throwIfUnchecked (com.google.common.base.Throwables.throwIfUnchecked)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableSet (com.google.common.collect.ImmutableSet)1