Examples with ConnectorMaterializedViewDefinition com.facebook.presto.spi.ConnectorMaterializedViewDefinition used on opensource projects

Example 6 with ConnectorMaterializedViewDefinition

use of com.facebook.presto.spi.ConnectorMaterializedViewDefinition in project presto by prestodb.

the class HiveMaterializedViewUtils method validateMaterializedViewPartitionColumns.

/**
 * Validate the partition columns of a materialized view to ensure 1) a materialized view is partitioned; 2) it has at least one partition
 * directly mapped to all base tables and 3) Outer join conditions have common partitions that are partitions in the view as well
 * <p>
 * A column is directly mapped to a base table column if it is derived directly or transitively from the base table column,
 * by only selecting a column or an aliased column without any function or operator applied.
 * For example, with SELECT column_b AS column_a, column_a is directly mapped to column_b.
 * With SELECT column_b + column_c AS column_a, column_a is not directly mapped to any column.
 * <p>
 * {@code viewToBaseColumnMap} only contains direct column mappings.
 */
public static void validateMaterializedViewPartitionColumns(SemiTransactionalHiveMetastore metastore, MetastoreContext metastoreContext, Table viewTable, ConnectorMaterializedViewDefinition viewDefinition) {
    SchemaTableName viewName = new SchemaTableName(viewTable.getDatabaseName(), viewTable.getTableName());
    Map<String, Map<SchemaTableName, String>> viewToBaseDirectColumnMap = viewDefinition.getDirectColumnMappingsAsMap();
    if (viewToBaseDirectColumnMap.isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, format("Materialized view %s must have at least one column directly defined by a base table column.", viewName));
    }
    List<Column> viewPartitions = viewTable.getPartitionColumns();
    if (viewPartitions.isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Unpartitioned materialized view is not supported.");
    }
    List<Table> baseTables = viewDefinition.getBaseTables().stream().map(baseTableName -> metastore.getTable(metastoreContext, baseTableName.getSchemaName(), baseTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(baseTableName))).collect(toImmutableList());
    Map<Table, List<Column>> baseTablePartitions = baseTables.stream().collect(toImmutableMap(table -> table, Table::getPartitionColumns));
    for (Table baseTable : baseTablePartitions.keySet()) {
        SchemaTableName schemaBaseTable = new SchemaTableName(baseTable.getDatabaseName(), baseTable.getTableName());
        if (!isCommonPartitionFound(schemaBaseTable, baseTablePartitions.get(baseTable), viewPartitions, viewToBaseDirectColumnMap)) {
            throw new PrestoException(NOT_SUPPORTED, format("Materialized view %s must have at least one partition column that exists in %s as well", viewName, baseTable.getTableName()));
        }
        if (viewDefinition.getBaseTablesOnOuterJoinSide().contains(schemaBaseTable) && viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(viewDefinition, baseTable).get().isEmpty()) {
            throw new PrestoException(NOT_SUPPORTED, format("Outer join conditions in Materialized view %s must have at least one common partition equality constraint", viewName));
        }
    }
}

Example 7 with ConnectorMaterializedViewDefinition

use of com.facebook.presto.spi.ConnectorMaterializedViewDefinition in project presto by prestodb.

the class HiveMaterializedViewUtils method viewToBaseTableOnOuterJoinSideIndirectMappedPartitions.

// Every table on outer join side, must have a partition which is in EQ clause and present in Materialized View as well.
// For a given base table, this function computes partition columns of Materialized View which are not directly mapped to base table,
// and are directly mapped to some other base table which is not on outer join side.
// For example:
// Materialized View: SELECT t1_a as t1.a, t2_a as t2.a FROM t1 LEFT JOIN t2 ON t1.a = t2.a, partitioned by [t1_a, t2_a]
// baseTable: t2, partitioned by [a]
// Output: t1_a -> t2.a
public static Optional<Map<String, String>> viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(ConnectorMaterializedViewDefinition viewDefinition, Table baseTable) {
    SchemaTableName schemaBaseTable = new SchemaTableName(baseTable.getDatabaseName(), baseTable.getTableName());
    if (!viewDefinition.getBaseTablesOnOuterJoinSide().contains(schemaBaseTable)) {
        return Optional.empty();
    }
    Map<String, String> viewToBaseIndirectMappedColumns = new HashMap<>();
    Map<String, Map<SchemaTableName, String>> columnMappings = viewDefinition.getColumnMappingsAsMap();
    Map<String, Map<SchemaTableName, String>> directColumnMappings = viewDefinition.getDirectColumnMappingsAsMap();
    for (String viewPartition : viewDefinition.getValidRefreshColumns().orElse(ImmutableList.of())) {
        String baseTablePartition = columnMappings.get(viewPartition).get(schemaBaseTable);
        // Check if it is a base table partition column
        if (baseTable.getPartitionColumns().stream().noneMatch(col -> col.getName().equals(baseTablePartition))) {
            continue;
        }
        // For e.g. in case of left outer join, we want to find partition which maps to left table
        if (directColumnMappings.get(viewPartition).keySet().stream().allMatch(e -> !e.equals(schemaBaseTable)) && directColumnMappings.get(viewPartition).keySet().stream().allMatch(t -> !viewDefinition.getBaseTablesOnOuterJoinSide().contains(t))) {
            viewToBaseIndirectMappedColumns.put(viewPartition, baseTablePartition);
        }
    }
    return Optional.of(viewToBaseIndirectMappedColumns);
}

Example 8 with ConnectorMaterializedViewDefinition

use of com.facebook.presto.spi.ConnectorMaterializedViewDefinition in project presto by prestodb.

the class HiveMetadata method createMaterializedView.

@Override
public void createMaterializedView(ConnectorSession session, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) {
    if (isExternalTable(viewMetadata.getProperties())) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, "Specifying external location for materialized view is not supported.");
    }
    Table basicTable = prepareTable(session, viewMetadata, MATERIALIZED_VIEW);
    viewDefinition = new ConnectorMaterializedViewDefinition(viewDefinition.getOriginalSql(), viewDefinition.getSchema(), viewDefinition.getTable(), viewDefinition.getBaseTables(), viewDefinition.getOwner(), viewDefinition.getColumnMappings(), viewDefinition.getBaseTablesOnOuterJoinSide(), Optional.of(getPartitionedBy(viewMetadata.getProperties())));
    Map<String, String> parameters = ImmutableMap.<String, String>builder().putAll(basicTable.getParameters()).put(PRESTO_MATERIALIZED_VIEW_FLAG, "true").build();
    Table viewTable = Table.builder(basicTable).setParameters(parameters).setViewOriginalText(Optional.of(encodeMaterializedViewData(MATERIALIZED_VIEW_JSON_CODEC.toJson(viewDefinition)))).setViewExpandedText(Optional.of("/* Presto Materialized View */")).build();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    validateMaterializedViewPartitionColumns(metastore, metastoreContext, viewTable, viewDefinition);
    try {
        PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(viewTable.getOwner());
        metastore.createTable(session, viewTable, principalPrivileges, Optional.empty(), ignoreExisting, new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
    } catch (TableAlreadyExistsException e) {
        throw new MaterializedViewAlreadyExistsException(e.getTableName());
    }
}

Example 9 with ConnectorMaterializedViewDefinition

use of com.facebook.presto.spi.ConnectorMaterializedViewDefinition in project presto by prestodb.

the class AddColumnTask method execute.

@Override
public ListenableFuture<?> execute(AddColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters, WarningCollector warningCollector) {
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (!tableHandle.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<ConnectorMaterializedViewDefinition> optionalMaterializedView = metadata.getMaterializedView(session, tableName);
    if (optionalMaterializedView.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and add column is not supported", tableName);
        }
        return immediateFuture(null);
    }
    ConnectorId connectorId = metadata.getCatalogHandle(session, tableName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName()));
    accessControl.checkCanAddColumns(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName);
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
    ColumnDefinition element = statement.getColumn();
    Type type;
    try {
        type = metadata.getType(parseTypeSignature(element.getType()));
    } catch (IllegalArgumentException e) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (type.equals(UNKNOWN)) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (columnHandles.containsKey(element.getName().getValue().toLowerCase(ENGLISH))) {
        if (!statement.isColumnNotExists()) {
            throw new SemanticException(COLUMN_ALREADY_EXISTS, statement, "Column '%s' already exists", element.getName());
        }
        return immediateFuture(null);
    }
    if (!element.isNullable() && !metadata.getConnectorCapabilities(session, connectorId).contains(NOT_NULL_COLUMN_CONSTRAINT)) {
        throw new SemanticException(NOT_SUPPORTED, element, "Catalog '%s' does not support NOT NULL for column '%s'", connectorId.getCatalogName(), element.getName());
    }
    Map<String, Expression> sqlProperties = mapFromProperties(element.getProperties());
    Map<String, Object> columnProperties = metadata.getColumnPropertyManager().getProperties(connectorId, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
    ColumnMetadata column = new ColumnMetadata(element.getName().getValue(), type, element.isNullable(), element.getComment().orElse(null), null, false, columnProperties);
    metadata.addColumn(session, tableHandle.get(), column);
    return immediateFuture(null);
}

Example 10 with ConnectorMaterializedViewDefinition

use of com.facebook.presto.spi.ConnectorMaterializedViewDefinition in project presto by prestodb.

the class DropColumnTask method execute.

@Override
public ListenableFuture<?> execute(DropColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters, WarningCollector warningCollector) {
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTable());
    Optional<TableHandle> tableHandleOptional = metadata.getTableHandle(session, tableName);
    if (!tableHandleOptional.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<ConnectorMaterializedViewDefinition> optionalMaterializedView = metadata.getMaterializedView(session, tableName);
    if (optionalMaterializedView.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and drop column is not supported", tableName);
        }
        return immediateFuture(null);
    }
    TableHandle tableHandle = tableHandleOptional.get();
    String column = statement.getColumn().getValue().toLowerCase(ENGLISH);
    accessControl.checkCanDropColumn(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName);
    ColumnHandle columnHandle = metadata.getColumnHandles(session, tableHandle).get(column);
    if (columnHandle == null) {
        if (!statement.isColumnExists()) {
            throw new SemanticException(MISSING_COLUMN, statement, "Column '%s' does not exist", column);
        }
        return immediateFuture(null);
    }
    if (metadata.getColumnMetadata(session, tableHandle, columnHandle).isHidden()) {
        throw new SemanticException(NOT_SUPPORTED, statement, "Cannot drop hidden column");
    }
    if (metadata.getTableMetadata(session, tableHandle).getColumns().stream().filter(info -> !info.isHidden()).count() <= 1) {
        throw new SemanticException(NOT_SUPPORTED, statement, "Cannot drop the only column in a table");
    }
    metadata.dropColumn(session, tableHandle, columnHandle);
    return immediateFuture(null);
}