Examples with HiveRelOptMaterialization org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization used on opensource projects

Example 1 with HiveRelOptMaterialization

use of org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization in project hive by apache.

the class HiveMaterializedViewUtils method augmentMaterializationWithTimeInformation.

/**
 * Method to enrich the materialization query contained in the input with
 * its invalidation.
 */
public static HiveRelOptMaterialization augmentMaterializationWithTimeInformation(HiveRelOptMaterialization materialization, String validTxnsList, ValidTxnWriteIdList materializationTxnList) throws LockException {
    // Extract tables used by the query which will in turn be used to generate
    // the corresponding txn write ids
    List<String> tablesUsed = new ArrayList<>();
    new RelVisitor() {

        @Override
        public void visit(RelNode node, int ordinal, RelNode parent) {
            if (node instanceof TableScan) {
                TableScan ts = (TableScan) node;
                tablesUsed.add(((RelOptHiveTable) ts.getTable()).getHiveTableMD().getFullyQualifiedName());
            }
            super.visit(node, ordinal, parent);
        }
    }.go(materialization.queryRel);
    ValidTxnWriteIdList currentTxnList = SessionState.get().getTxnMgr().getValidWriteIds(tablesUsed, validTxnsList);
    // Augment
    final RexBuilder rexBuilder = materialization.queryRel.getCluster().getRexBuilder();
    final HepProgramBuilder augmentMaterializationProgram = new HepProgramBuilder().addRuleInstance(new HiveAugmentMaterializationRule(rexBuilder, currentTxnList, materializationTxnList));
    final HepPlanner augmentMaterializationPlanner = new HepPlanner(augmentMaterializationProgram.build());
    augmentMaterializationPlanner.setRoot(materialization.queryRel);
    final RelNode modifiedQueryRel = augmentMaterializationPlanner.findBestExp();
    return new HiveRelOptMaterialization(materialization.tableRel, modifiedQueryRel, null, materialization.qualifiedTableName, materialization.getScope(), materialization.getRebuildMode());
}

Example 2 with HiveRelOptMaterialization

use of org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization in project hive by apache.

the class HiveMaterializedViewUtils method deriveGroupingSetsMaterializedViews.

/**
 * If a materialization does not contain grouping sets, it returns the materialization
 * itself. Otherwise, it will create one materialization for each grouping set.
 * For each grouping set, the query for the materialization will consist of the group by
 * columns in the grouping set, followed by a projection to recreate the literal null
 * values. The view scan will consist of the scan over the materialization followed by a
 * filter on the grouping id value corresponding to that grouping set.
 */
public static List<HiveRelOptMaterialization> deriveGroupingSetsMaterializedViews(HiveRelOptMaterialization materialization) {
    final RelNode query = materialization.queryRel;
    final Project project;
    final Aggregate aggregate;
    if (query instanceof Aggregate) {
        project = null;
        aggregate = (Aggregate) query;
    } else if (query instanceof Project && query.getInput(0) instanceof Aggregate) {
        project = (Project) query;
        aggregate = (Aggregate) query.getInput(0);
    } else {
        project = null;
        aggregate = null;
    }
    if (aggregate == null) {
        // Not an aggregate materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    if (aggregate.getGroupType() == Group.SIMPLE) {
        // Not a grouping sets materialized view, return original materialization
        return Collections.singletonList(materialization);
    }
    int aggregateGroupingIdIndex = -1;
    for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
        if (aggregate.getAggCallList().get(i).getAggregation() == HiveGroupingID.INSTANCE) {
            aggregateGroupingIdIndex = aggregate.getGroupCount() + i;
            break;
        }
    }
    Preconditions.checkState(aggregateGroupingIdIndex != -1);
    int projectGroupingIdIndex = -1;
    if (project != null) {
        for (int i = 0; i < project.getProjects().size(); i++) {
            RexNode expr = project.getProjects().get(i);
            if (expr instanceof RexInputRef) {
                RexInputRef ref = (RexInputRef) expr;
                if (ref.getIndex() == aggregateGroupingIdIndex) {
                    // Grouping id is present
                    projectGroupingIdIndex = i;
                    break;
                }
            }
        }
        if (projectGroupingIdIndex == -1) {
            // Grouping id is not present, return original materialization
            return Collections.singletonList(materialization);
        }
    }
    // Create multiple materializations
    final List<HiveRelOptMaterialization> materializationList = new ArrayList<>();
    final RelBuilder builder = HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<AggregateCall> aggregateCalls = new ArrayList<>(aggregate.getAggCallList());
    aggregateCalls.remove(aggregateGroupingIdIndex - aggregate.getGroupCount());
    for (ImmutableBitSet targetGroupSet : aggregate.getGroupSets()) {
        // Compute the grouping id value
        long groupingIdValue = convert(targetGroupSet, aggregate.getGroupSet());
        // First we modify the MV query
        Aggregate newAggregate = aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(), targetGroupSet, null, aggregateCalls);
        builder.push(newAggregate);
        List<RexNode> exprs = new ArrayList<>();
        for (int pos = 0; pos < aggregate.getGroupCount(); pos++) {
            int ref = aggregate.getGroupSet().nth(pos);
            if (targetGroupSet.get(ref)) {
                exprs.add(rexBuilder.makeInputRef(newAggregate, targetGroupSet.indexOf(ref)));
            } else {
                exprs.add(rexBuilder.makeNullLiteral(aggregate.getRowType().getFieldList().get(pos).getType()));
            }
        }
        int pos = targetGroupSet.cardinality();
        for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
            if (aggregateCall.getAggregation() == HiveGroupingID.INSTANCE) {
                exprs.add(rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
            } else {
                exprs.add(rexBuilder.makeInputRef(newAggregate, pos++));
            }
        }
        if (project != null) {
            // Include projections from top operator
            Project bottomProject = (Project) builder.project(exprs, ImmutableList.of(), true).build();
            List<RexNode> newNodes = RelOptUtil.pushPastProject(project.getProjects(), bottomProject);
            builder.push(bottomProject.getInput()).project(newNodes);
        } else {
            builder.project(exprs);
        }
        final RelNode newQueryRel = builder.build();
        // Second we modify the MV scan
        builder.push(materialization.tableRel);
        RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexBuilder.makeInputRef(materialization.tableRel, project != null ? projectGroupingIdIndex : aggregateGroupingIdIndex), rexBuilder.makeBigintLiteral(new BigDecimal(groupingIdValue)));
        builder.filter(condition);
        final RelNode newTableRel = builder.build();
        final Table scanTable = extractTable(materialization);
        materializationList.add(new HiveRelOptMaterialization(newTableRel, newQueryRel, null, ImmutableList.of(scanTable.getDbName(), scanTable.getTableName(), "#" + materializationList.size()), materialization.getScope(), materialization.getRebuildMode()));
    }
    return materializationList;
}

Example 3 with HiveRelOptMaterialization

use of org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization in project hive by apache.

the class ShowMaterializedViewsFormatter method formatIncrementalRebuildMode.

@NotNull
private static String formatIncrementalRebuildMode(Table materializedView) {
    String incrementalRebuild;
    HiveRelOptMaterialization relOptMaterialization = HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(materializedView.getDbName(), materializedView.getTableName(), ALL);
    if (relOptMaterialization == null || relOptMaterialization.getRebuildMode() == UNKNOWN) {
        incrementalRebuild = "Unknown";
    } else {
        switch(relOptMaterialization.getRebuildMode()) {
            case AVAILABLE:
                incrementalRebuild = "Available";
                break;
            case INSERT_ONLY:
                incrementalRebuild = "Available in presence of insert operations only";
                break;
            case NOT_AVAILABLE:
                incrementalRebuild = "Not available";
                break;
            default:
                incrementalRebuild = "Unknown";
                break;
        }
    }
    return incrementalRebuild;
}