Examples with ConnectorMaterializedViewDefinition io.trino.spi.connector.ConnectorMaterializedViewDefinition used on opensource projects

Example 6 with ConnectorMaterializedViewDefinition

use of io.trino.spi.connector.ConnectorMaterializedViewDefinition in project trino by trinodb.

the class IcebergMetadata method getMaterializedViewToken.

private Map<String, Optional<TableToken>> getMaterializedViewToken(ConnectorSession session, SchemaTableName name) {
    Map<String, Optional<TableToken>> viewToken = new HashMap<>();
    Optional<ConnectorMaterializedViewDefinition> materializedViewDefinition = getMaterializedView(session, name);
    if (materializedViewDefinition.isEmpty()) {
        return viewToken;
    }
    SchemaTableName storageTableName = materializedViewDefinition.get().getStorageTable().map(CatalogSchemaTableName::getSchemaTableName).orElseThrow(() -> new IllegalStateException("Storage table missing in definition of materialized view " + name));
    Table icebergTable = catalog.loadTable(session, storageTableName);
    String dependsOnTables = icebergTable.currentSnapshot().summary().getOrDefault(DEPENDS_ON_TABLES, "");
    if (!dependsOnTables.isEmpty()) {
        Map<String, String> tableToSnapshotIdMap = Splitter.on(',').withKeyValueSeparator('=').split(dependsOnTables);
        for (Map.Entry<String, String> entry : tableToSnapshotIdMap.entrySet()) {
            viewToken.put(entry.getKey(), Optional.of(new TableToken(Long.parseLong(entry.getValue()))));
        }
    }
    return viewToken;
}

Example 7 with ConnectorMaterializedViewDefinition

use of io.trino.spi.connector.ConnectorMaterializedViewDefinition in project trino by trinodb.

the class TestRefreshMaterializedView method createQueryRunner.

@Override
protected QueryRunner createQueryRunner() throws Exception {
    Session session = testSessionBuilder().setCatalog("mock").setSchema("default").build();
    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
    queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder().withListSchemaNames(connectionSession -> ImmutableList.of("default")).withGetColumns(schemaTableName -> ImmutableList.of(new ColumnMetadata("nationkey", BIGINT))).withGetTableHandle((connectorSession, tableName) -> new MockConnectorTableHandle(tableName)).withGetMaterializedViews((connectorSession, schemaTablePrefix) -> ImmutableMap.of(new SchemaTableName("default", "delegate_refresh_to_connector"), new ConnectorMaterializedViewDefinition("SELECT nationkey FROM mock.default.test_table", Optional.of(new CatalogSchemaTableName("mock", "default", "test_storage")), Optional.of("mock"), Optional.of("default"), ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("nationkey", BIGINT.getTypeId())), Optional.empty(), Optional.of("alice"), ImmutableMap.of()))).withDelegateMaterializedViewRefreshToConnector((connectorSession, schemaTableName) -> true).withRefreshMaterializedView(((connectorSession, schemaTableName) -> {
        startRefreshMaterializedView.set(null);
        SettableFuture<Void> refreshMaterializedView = SettableFuture.create();
        finishRefreshMaterializedView.addListener(() -> refreshMaterializedView.set(null), directExecutor());
        addExceptionCallback(refreshMaterializedView, () -> refreshInterrupted.set(null));
        return toCompletableFuture(refreshMaterializedView);
    })).build()));
    queryRunner.createCatalog("mock", "mock");
    return queryRunner;
}

Example 8 with ConnectorMaterializedViewDefinition

use of io.trino.spi.connector.ConnectorMaterializedViewDefinition in project trino by trinodb.

the class TrinoHiveCatalog method createMaterializedView.

@Override
public void createMaterializedView(ConnectorSession session, SchemaTableName schemaViewName, ConnectorMaterializedViewDefinition definition, boolean replace, boolean ignoreExisting) {
    Optional<io.trino.plugin.hive.metastore.Table> existing = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
    // It's a create command where the materialized view already exists and 'if not exists' clause is not specified
    if (!replace && existing.isPresent()) {
        if (ignoreExisting) {
            return;
        }
        throw new TrinoException(ALREADY_EXISTS, "Materialized view already exists: " + schemaViewName);
    }
    // Generate a storage table name and create a storage table. The properties in the definition are table properties for the
    // storage table as indicated in the materialized view definition.
    String storageTableName = "st_" + randomUUID().toString().replace("-", "");
    Map<String, Object> storageTableProperties = new HashMap<>(definition.getProperties());
    storageTableProperties.putIfAbsent(FILE_FORMAT_PROPERTY, DEFAULT_FILE_FORMAT_DEFAULT);
    SchemaTableName storageTable = new SchemaTableName(schemaViewName.getSchemaName(), storageTableName);
    List<ColumnMetadata> columns = definition.getColumns().stream().map(column -> new ColumnMetadata(column.getName(), typeManager.getType(column.getType()))).collect(toImmutableList());
    ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(storageTable, columns, storageTableProperties, Optional.empty());
    Transaction transaction = IcebergUtil.newCreateTableTransaction(this, tableMetadata, session);
    transaction.newAppend().commit();
    transaction.commitTransaction();
    // Create a view indicating the storage table
    Map<String, String> viewProperties = ImmutableMap.<String, String>builder().put(PRESTO_QUERY_ID_NAME, session.getQueryId()).put(STORAGE_TABLE, storageTableName).put(PRESTO_VIEW_FLAG, "true").put(TRINO_CREATED_BY, TRINO_CREATED_BY_VALUE).put(TABLE_COMMENT, ICEBERG_MATERIALIZED_VIEW_COMMENT).buildOrThrow();
    Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty());
    io.trino.plugin.hive.metastore.Table.Builder tableBuilder = io.trino.plugin.hive.metastore.Table.builder().setDatabaseName(schemaViewName.getSchemaName()).setTableName(schemaViewName.getTableName()).setOwner(isUsingSystemSecurity ? Optional.empty() : Optional.of(session.getUser())).setTableType(VIRTUAL_VIEW.name()).setDataColumns(ImmutableList.of(dummyColumn)).setPartitionColumns(ImmutableList.of()).setParameters(viewProperties).withStorage(storage -> storage.setStorageFormat(VIEW_STORAGE_FORMAT)).withStorage(storage -> storage.setLocation("")).setViewOriginalText(Optional.of(encodeMaterializedViewData(fromConnectorMaterializedViewDefinition(definition)))).setViewExpandedText(Optional.of("/* Presto Materialized View */"));
    io.trino.plugin.hive.metastore.Table table = tableBuilder.build();
    PrincipalPrivileges principalPrivileges = isUsingSystemSecurity ? NO_PRIVILEGES : buildInitialPrivilegeSet(session.getUser());
    if (existing.isPresent() && replace) {
        // drop the current storage table
        String oldStorageTable = existing.get().getParameters().get(STORAGE_TABLE);
        if (oldStorageTable != null) {
            metastore.dropTable(schemaViewName.getSchemaName(), oldStorageTable, true);
        }
        // Replace the existing view definition
        metastore.replaceTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), table, principalPrivileges);
        return;
    }
    // create the view definition
    metastore.createTable(table, principalPrivileges);
}

Example 9 with ConnectorMaterializedViewDefinition

use of io.trino.spi.connector.ConnectorMaterializedViewDefinition in project trino by trinodb.

the class TrinoHiveCatalog method doGetMaterializedView.

private Optional<ConnectorMaterializedViewDefinition> doGetMaterializedView(ConnectorSession session, SchemaTableName schemaViewName) {
    Optional<io.trino.plugin.hive.metastore.Table> tableOptional = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
    if (tableOptional.isEmpty()) {
        return Optional.empty();
    }
    io.trino.plugin.hive.metastore.Table table = tableOptional.get();
    if (!isPrestoView(table) || !isHiveOrPrestoView(table) || !table.getParameters().containsKey(STORAGE_TABLE)) {
        return Optional.empty();
    }
    io.trino.plugin.hive.metastore.Table materializedView = tableOptional.get();
    String storageTable = materializedView.getParameters().get(STORAGE_TABLE);
    checkState(storageTable != null, "Storage table missing in definition of materialized view " + schemaViewName);
    IcebergMaterializedViewDefinition definition = decodeMaterializedViewData(materializedView.getViewOriginalText().orElseThrow(() -> new TrinoException(HIVE_INVALID_METADATA, "No view original text: " + schemaViewName)));
    Table icebergTable;
    try {
        icebergTable = loadTable(session, new SchemaTableName(schemaViewName.getSchemaName(), storageTable));
    } catch (RuntimeException e) {
        // The materialized view could be removed concurrently. This may manifest in a number of ways, e.g.
        // - io.trino.spi.connector.TableNotFoundException
        // - org.apache.iceberg.exceptions.NotFoundException when accessing manifest file
        // - other failures when reading storage table's metadata files
        // Retry, as we're catching broadly.
        metastore.invalidateTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
        metastore.invalidateTable(schemaViewName.getSchemaName(), storageTable);
        throw new MaterializedViewMayBeBeingRemovedException(e);
    }
    ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
    properties.put(FILE_FORMAT_PROPERTY, IcebergUtil.getFileFormat(icebergTable));
    if (!icebergTable.spec().fields().isEmpty()) {
        properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec()));
    }
    return Optional.of(new ConnectorMaterializedViewDefinition(definition.getOriginalSql(), Optional.of(new CatalogSchemaTableName(catalogName.toString(), new SchemaTableName(schemaViewName.getSchemaName(), storageTable))), definition.getCatalog(), definition.getSchema(), definition.getColumns().stream().map(column -> new ConnectorMaterializedViewDefinition.Column(column.getName(), column.getType())).collect(toImmutableList()), definition.getComment(), materializedView.getOwner(), properties.buildOrThrow()));
}

Example 10 with ConnectorMaterializedViewDefinition

use of io.trino.spi.connector.ConnectorMaterializedViewDefinition in project trino by trinodb.

the class TestingMetadata method renameMaterializedView.

@Override
public void renameMaterializedView(ConnectorSession session, SchemaTableName source, SchemaTableName target) {
    // TODO: use locking to do this properly
    ConnectorMaterializedViewDefinition materializedView = getMaterializedView(session, source).orElseThrow();
    if (materializedViews.putIfAbsent(target, materializedView) != null) {
        throw new IllegalArgumentException("Target materialized view already exists: " + target);
    }
    materializedViews.remove(source, materializedView);
}