Search in sources :

Example 1 with IcebergMaterializedViewDefinition

use of io.trino.plugin.iceberg.IcebergMaterializedViewDefinition in project trino by trinodb.

the class TrinoHiveCatalog method doGetMaterializedView.

private Optional<ConnectorMaterializedViewDefinition> doGetMaterializedView(ConnectorSession session, SchemaTableName schemaViewName) {
    Optional<io.trino.plugin.hive.metastore.Table> tableOptional = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
    if (tableOptional.isEmpty()) {
        return Optional.empty();
    }
    io.trino.plugin.hive.metastore.Table table = tableOptional.get();
    if (!isPrestoView(table) || !isHiveOrPrestoView(table) || !table.getParameters().containsKey(STORAGE_TABLE)) {
        return Optional.empty();
    }
    io.trino.plugin.hive.metastore.Table materializedView = tableOptional.get();
    String storageTable = materializedView.getParameters().get(STORAGE_TABLE);
    checkState(storageTable != null, "Storage table missing in definition of materialized view " + schemaViewName);
    IcebergMaterializedViewDefinition definition = decodeMaterializedViewData(materializedView.getViewOriginalText().orElseThrow(() -> new TrinoException(HIVE_INVALID_METADATA, "No view original text: " + schemaViewName)));
    Table icebergTable;
    try {
        icebergTable = loadTable(session, new SchemaTableName(schemaViewName.getSchemaName(), storageTable));
    } catch (RuntimeException e) {
        // The materialized view could be removed concurrently. This may manifest in a number of ways, e.g.
        // - io.trino.spi.connector.TableNotFoundException
        // - org.apache.iceberg.exceptions.NotFoundException when accessing manifest file
        // - other failures when reading storage table's metadata files
        // Retry, as we're catching broadly.
        metastore.invalidateTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
        metastore.invalidateTable(schemaViewName.getSchemaName(), storageTable);
        throw new MaterializedViewMayBeBeingRemovedException(e);
    }
    ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
    properties.put(FILE_FORMAT_PROPERTY, IcebergUtil.getFileFormat(icebergTable));
    if (!icebergTable.spec().fields().isEmpty()) {
        properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec()));
    }
    return Optional.of(new ConnectorMaterializedViewDefinition(definition.getOriginalSql(), Optional.of(new CatalogSchemaTableName(catalogName.toString(), new SchemaTableName(schemaViewName.getSchemaName(), storageTable))), definition.getCatalog(), definition.getSchema(), definition.getColumns().stream().map(column -> new ConnectorMaterializedViewDefinition.Column(column.getName(), column.getType())).collect(toImmutableList()), definition.getComment(), materializedView.getOwner(), properties.buildOrThrow()));
}
Also used : FileSystem(org.apache.hadoop.fs.FileSystem) Throwables.throwIfUnchecked(com.google.common.base.Throwables.throwIfUnchecked) ICEBERG_TABLE_TYPE_VALUE(org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE) TABLE_TYPE_PROP(org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Column(io.trino.plugin.hive.metastore.Column) TABLE_NOT_FOUND(io.trino.spi.StandardErrorCode.TABLE_NOT_FOUND) Duration(java.time.Duration) Map(java.util.Map) ViewNotFoundException(io.trino.spi.connector.ViewNotFoundException) TABLE_COMMENT(io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) BaseTable(org.apache.iceberg.BaseTable) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Schema(org.apache.iceberg.Schema) IcebergUtil.validateTableCanBeDropped(io.trino.plugin.iceberg.IcebergUtil.validateTableCanBeDropped) SchemaTableName(io.trino.spi.connector.SchemaTableName) HiveMetadata(io.trino.plugin.hive.HiveMetadata) Stream(java.util.stream.Stream) TrinoPrincipal(io.trino.spi.security.TrinoPrincipal) STORAGE_TABLE(io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) CatalogUtil.dropTableData(org.apache.iceberg.CatalogUtil.dropTableData) HivePrincipal(io.trino.plugin.hive.metastore.HivePrincipal) HiveUtil(io.trino.plugin.hive.util.HiveUtil) HiveViewNotSupportedException(io.trino.plugin.hive.HiveViewNotSupportedException) IcebergMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) HIVE_INVALID_METADATA(io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) SCHEMA_NOT_EMPTY(io.trino.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) FILE_FORMAT_PROPERTY(io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY) AbstractTrinoCatalog(io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog) ViewReaderUtil.isPrestoView(io.trino.plugin.hive.ViewReaderUtil.isPrestoView) Table(org.apache.iceberg.Table) IOException(java.io.IOException) ConnectorSession(io.trino.spi.connector.ConnectorSession) CatalogName(io.trino.plugin.base.CatalogName) Failsafe(net.jodah.failsafe.Failsafe) HiveUtil.isHiveSystemSchema(io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema) ChronoUnit(java.time.temporal.ChronoUnit) ViewReaderUtil.isHiveOrPrestoView(io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView) MaterializedViewNotFoundException(io.trino.spi.connector.MaterializedViewNotFoundException) IcebergSchemaProperties.getSchemaLocation(io.trino.plugin.iceberg.IcebergSchemaProperties.getSchemaLocation) IcebergMaterializedViewDefinition.decodeMaterializedViewData(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.decodeMaterializedViewData) Database(io.trino.plugin.hive.metastore.Database) HiveSchemaProperties(io.trino.plugin.hive.HiveSchemaProperties) ConnectorMaterializedViewDefinition(io.trino.spi.connector.ConnectorMaterializedViewDefinition) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) TableMetadata(org.apache.iceberg.TableMetadata) IcebergUtil(io.trino.plugin.iceberg.IcebergUtil) Locale(java.util.Locale) ALREADY_EXISTS(io.trino.spi.StandardErrorCode.ALREADY_EXISTS) Path(org.apache.hadoop.fs.Path) ViewReaderUtil.encodeViewData(io.trino.plugin.hive.ViewReaderUtil.encodeViewData) ConnectorViewDefinition(io.trino.spi.connector.ConnectorViewDefinition) ImmutableSet(com.google.common.collect.ImmutableSet) HiveWriteUtils.getTableDefaultLocation(io.trino.plugin.hive.util.HiveWriteUtils.getTableDefaultLocation) ImmutableMap(com.google.common.collect.ImmutableMap) IcebergTableOperationsProvider(io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) VIEW_STORAGE_FORMAT(io.trino.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) TrinoException(io.trino.spi.TrinoException) RetryPolicy(net.jodah.failsafe.RetryPolicy) Collectors(java.util.stream.Collectors) Preconditions.checkState(com.google.common.base.Preconditions.checkState) PRESTO_VIEW_FLAG(io.trino.plugin.hive.ViewReaderUtil.PRESTO_VIEW_FLAG) INVALID_SCHEMA_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) List(java.util.List) IcebergUtil.loadIcebergTable(io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable) MetastoreUtil.buildInitialPrivilegeSet(io.trino.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet) PartitionSpec(org.apache.iceberg.PartitionSpec) Optional(java.util.Optional) Logger(io.airlift.log.Logger) PartitionFields.toPartitionFields(io.trino.plugin.iceberg.PartitionFields.toPartitionFields) ColumnIdentity(io.trino.plugin.iceberg.ColumnIdentity) ViewReaderUtil(io.trino.plugin.hive.ViewReaderUtil) HashMap(java.util.HashMap) IcebergUtil.getIcebergTableWithMetadata(io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata) PARTITIONING_PROPERTY(io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) CachingHiveMetastore(io.trino.plugin.hive.metastore.cache.CachingHiveMetastore) VIRTUAL_VIEW(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW) IcebergMaterializedViewDefinition.encodeMaterializedViewData(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.encodeMaterializedViewData) NO_PRIVILEGES(io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES) ViewAlreadyExistsException(io.trino.plugin.hive.ViewAlreadyExistsException) HIVE_STRING(io.trino.plugin.hive.HiveType.HIVE_STRING) IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition) UUID.randomUUID(java.util.UUID.randomUUID) Transaction(org.apache.iceberg.Transaction) DEFAULT_FILE_FORMAT_DEFAULT(org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) TypeManager(io.trino.spi.type.TypeManager) IcebergMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition) BaseTable(org.apache.iceberg.BaseTable) Table(org.apache.iceberg.Table) IcebergUtil.loadIcebergTable(io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable) ConnectorMaterializedViewDefinition(io.trino.spi.connector.ConnectorMaterializedViewDefinition) IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition(io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition) SchemaTableName(io.trino.spi.connector.SchemaTableName) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) ImmutableMap(com.google.common.collect.ImmutableMap) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) Column(io.trino.plugin.hive.metastore.Column) TrinoException(io.trino.spi.TrinoException)

Aggregations

Preconditions.checkState (com.google.common.base.Preconditions.checkState)1 Throwables.throwIfUnchecked (com.google.common.base.Throwables.throwIfUnchecked)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Logger (io.airlift.log.Logger)1 CatalogName (io.trino.plugin.base.CatalogName)1 HdfsEnvironment (io.trino.plugin.hive.HdfsEnvironment)1 HdfsContext (io.trino.plugin.hive.HdfsEnvironment.HdfsContext)1 HIVE_INVALID_METADATA (io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA)1 HiveMetadata (io.trino.plugin.hive.HiveMetadata)1 STORAGE_TABLE (io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE)1 TABLE_COMMENT (io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT)1 HiveSchemaProperties (io.trino.plugin.hive.HiveSchemaProperties)1 HIVE_STRING (io.trino.plugin.hive.HiveType.HIVE_STRING)1 HiveViewNotSupportedException (io.trino.plugin.hive.HiveViewNotSupportedException)1 TableAlreadyExistsException (io.trino.plugin.hive.TableAlreadyExistsException)1 ViewAlreadyExistsException (io.trino.plugin.hive.ViewAlreadyExistsException)1 ViewReaderUtil (io.trino.plugin.hive.ViewReaderUtil)1