Examples with RelOptMaterialization org.apache.calcite.plan.RelOptMaterialization used on opensource projects

Example 6 with RelOptMaterialization

use of org.apache.calcite.plan.RelOptMaterialization in project hive by apache.

the class Hive method getValidMaterializedViews.

/**
 * Get the materialized views that have been enabled for rewriting from the
 * metastore. If the materialized view is in the cache, we do not need to
 * parse it to generate a logical plan for the rewriting. Instead, we
 * return the version present in the cache. Further, information provided
 * by the invalidation cache is useful to know whether a materialized view
 * can be used for rewriting or not.
 *
 * @return the list of materialized views available for rewriting
 * @throws HiveException
 */
public List<RelOptMaterialization> getValidMaterializedViews(boolean materializedViewRebuild) throws HiveException {
    final long defaultDiff = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW, TimeUnit.MILLISECONDS);
    final long currentTime = System.currentTimeMillis();
    try {
        // Final result
        List<RelOptMaterialization> result = new ArrayList<>();
        for (String dbName : getMSC().getAllDatabases()) {
            // From metastore (for security)
            List<String> materializedViewNames = getMaterializedViewsForRewriting(dbName);
            if (materializedViewNames.isEmpty()) {
                // Bail out: empty list
                continue;
            }
            List<Table> materializedViewTables = getTableObjects(dbName, materializedViewNames);
            Map<String, Materialization> databaseInvalidationInfo = getMSC().getMaterializationsInvalidationInfo(dbName, materializedViewNames);
            for (Table materializedViewTable : materializedViewTables) {
                // Check whether the materialized view is invalidated
                Materialization materializationInvalidationInfo = databaseInvalidationInfo.get(materializedViewTable.getTableName());
                if (materializationInvalidationInfo == null) {
                    LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " ignored for rewriting as there was no information loaded in the invalidation cache");
                    continue;
                }
                // Check if materialization defined its own invalidation time window
                String timeWindowString = materializedViewTable.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW);
                long diff = org.apache.commons.lang.StringUtils.isEmpty(timeWindowString) ? defaultDiff : HiveConf.toTime(timeWindowString, HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), TimeUnit.MILLISECONDS);
                long invalidationTime = materializationInvalidationInfo.getInvalidationTime();
                // If we are doing a rebuild, we do not consider outdated materialized views either.
                if (diff == 0L || materializedViewRebuild) {
                    if (invalidationTime != 0L) {
                        // If parameter is zero, materialized view cannot be outdated at all
                        LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " ignored for rewriting as its contents are outdated");
                        continue;
                    }
                } else {
                    if (invalidationTime != 0 && invalidationTime > currentTime - diff) {
                        LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " ignored for rewriting as its contents are outdated");
                        continue;
                    }
                }
                // It passed the test, load
                RelOptMaterialization materialization = HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(dbName, materializedViewTable.getTableName());
                if (materialization != null) {
                    RelOptHiveTable cachedMaterializedViewTable = (RelOptHiveTable) materialization.tableRel.getTable();
                    if (cachedMaterializedViewTable.getHiveTableMD().getCreateTime() == materializedViewTable.getCreateTime()) {
                        // It is in the cache and up to date
                        result.add(materialization);
                        continue;
                    }
                }
                // or it is not up to date.
                if (HiveMaterializedViewsRegistry.get().isInitialized()) {
                    // But the registry was fully initialized, thus we need to add it
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " was not in the cache");
                    }
                    materialization = HiveMaterializedViewsRegistry.get().createMaterializedView(conf, materializedViewTable);
                    if (materialization != null) {
                        result.add(materialization);
                    }
                } else {
                    // Otherwise the registry has not been initialized, skip for the time being
                    if (LOG.isWarnEnabled()) {
                        LOG.info("Materialized view " + materializedViewTable.getFullyQualifiedName() + " was skipped " + "because cache has not been loaded yet");
                    }
                }
            }
        }
        return result;
    } catch (Exception e) {
        throw new HiveException(e);
    }
}

Example 7 with RelOptMaterialization

use of org.apache.calcite.plan.RelOptMaterialization in project hive by apache.

the class HiveMaterializedViewsRegistry method addMaterializedView.

/**
 * Adds the materialized view to the cache.
 *
 * @param materializedViewTable the materialized view
 */
private RelOptMaterialization addMaterializedView(HiveConf conf, Table materializedViewTable, OpType opType) {
    // Bail out if it is not enabled for rewriting
    if (!materializedViewTable.isRewriteEnabled()) {
        return null;
    }
    // We are going to create the map for each view in the given database
    ConcurrentMap<String, RelOptMaterialization> cq = new ConcurrentHashMap<String, RelOptMaterialization>();
    if (!dummy) {
        // If we are caching the MV, we include it in the cache
        final ConcurrentMap<String, RelOptMaterialization> prevCq = materializedViews.putIfAbsent(materializedViewTable.getDbName(), cq);
        if (prevCq != null) {
            cq = prevCq;
        }
    }
    // Start the process to add MV to the cache
    // First we parse the view query and create the materialization object
    final String viewQuery = materializedViewTable.getViewExpandedText();
    final RelNode viewScan = createMaterializedViewScan(conf, materializedViewTable);
    if (viewScan == null) {
        LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + " ignored; error creating view replacement");
        return null;
    }
    final RelNode queryRel = parseQuery(conf, viewQuery);
    if (queryRel == null) {
        LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + " ignored; error parsing original query");
        return null;
    }
    RelOptMaterialization materialization = new RelOptMaterialization(viewScan, queryRel, null, viewScan.getTable().getQualifiedName());
    if (opType == OpType.CREATE) {
        // You store the materialized view
        cq.put(materializedViewTable.getTableName(), materialization);
    } else {
        // For LOAD, you only add it if it does exist as you might be loading an outdated MV
        cq.putIfAbsent(materializedViewTable.getTableName(), materialization);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Created materialized view for rewriting: " + viewScan.getTable().getQualifiedName());
    }
    return materialization;
}

Example 8 with RelOptMaterialization

use of org.apache.calcite.plan.RelOptMaterialization in project hive by apache.

the class HiveMaterializedViewsRegistry method refreshMaterializedView.

/**
 * Update the materialized view in the registry (if materialized view exists).
 */
public void refreshMaterializedView(HiveConf conf, Table materializedViewTable) {
    RelOptMaterialization cached = materializedViewsCache.get(materializedViewTable.getDbName(), materializedViewTable.getTableName());
    if (cached == null) {
        return;
    }
    Table cachedTable = HiveMaterializedViewUtils.extractTable(cached);
    refreshMaterializedView(conf, cachedTable, materializedViewTable);
}

Example 9 with RelOptMaterialization

use of org.apache.calcite.plan.RelOptMaterialization in project calcite by apache.

the class Prepare method optimize.

/**
 * Optimizes a query plan.
 *
 * @param root Root of relational expression tree
 * @param materializations Tables known to be populated with a given query
 * @param lattices Lattices
 * @return an equivalent optimized relational expression
 */
protected RelRoot optimize(RelRoot root, final List<Materialization> materializations, final List<CalciteSchema.LatticeEntry> lattices) {
    final RelOptPlanner planner = root.rel.getCluster().getPlanner();
    final DataContext dataContext = context.getDataContext();
    planner.setExecutor(new RexExecutorImpl(dataContext));
    final List<RelOptMaterialization> materializationList = new ArrayList<>();
    for (Materialization materialization : materializations) {
        List<String> qualifiedTableName = materialization.materializedTable.path();
        materializationList.add(new RelOptMaterialization(materialization.tableRel, materialization.queryRel, materialization.starRelOptTable, qualifiedTableName));
    }
    final List<RelOptLattice> latticeList = new ArrayList<>();
    for (CalciteSchema.LatticeEntry lattice : lattices) {
        final CalciteSchema.TableEntry starTable = lattice.getStarTable();
        final JavaTypeFactory typeFactory = context.getTypeFactory();
        final RelOptTableImpl starRelOptTable = RelOptTableImpl.create(catalogReader, starTable.getTable().getRowType(typeFactory), starTable, null);
        latticeList.add(new RelOptLattice(lattice.getLattice(), starRelOptTable));
    }
    final RelTraitSet desiredTraits = getDesiredRootTraitSet(root);
    // Work around
    // [CALCITE-1774] Allow rules to be registered during planning process
    // by briefly creating each kind of physical table to let it register its
    // rules. The problem occurs when plans are created via RelBuilder, not
    // the usual process (SQL and SqlToRelConverter.Config.isConvertTableAccess
    // = true).
    final RelVisitor visitor = new RelVisitor() {

        @Override
        public void visit(RelNode node, int ordinal, RelNode parent) {
            if (node instanceof TableScan) {
                final RelOptCluster cluster = node.getCluster();
                final RelOptTable.ToRelContext context = RelOptUtil.getContext(cluster);
                final RelNode r = node.getTable().toRel(context);
                planner.registerClass(r);
            }
            super.visit(node, ordinal, parent);
        }
    };
    visitor.go(root.rel);
    final Program program = getProgram();
    final RelNode rootRel4 = program.run(planner, root.rel, desiredTraits, materializationList, latticeList);
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Plan after physical tweaks: {}", RelOptUtil.toString(rootRel4, SqlExplainLevel.ALL_ATTRIBUTES));
    }
    return root.withRel(rootRel4);
}

Example 10 with RelOptMaterialization

use of org.apache.calcite.plan.RelOptMaterialization in project calcite by apache.

the class MaterializedViewFilterScanRule method apply.

protected void apply(RelOptRuleCall call, Filter filter, TableScan scan) {
    RelOptPlanner planner = call.getPlanner();
    List<RelOptMaterialization> materializations = (planner instanceof VolcanoPlanner) ? ((VolcanoPlanner) planner).getMaterializations() : ImmutableList.<RelOptMaterialization>of();
    if (!materializations.isEmpty()) {
        RelNode root = filter.copy(filter.getTraitSet(), Collections.singletonList((RelNode) scan));
        List<RelOptMaterialization> applicableMaterializations = RelOptMaterializations.getApplicableMaterializations(root, materializations);
        for (RelOptMaterialization materialization : applicableMaterializations) {
            if (RelOptUtil.areRowTypesEqual(scan.getRowType(), materialization.queryRel.getRowType(), false)) {
                RelNode target = materialization.queryRel;
                final HepPlanner hepPlanner = new HepPlanner(program, planner.getContext());
                hepPlanner.setRoot(target);
                target = hepPlanner.findBestExp();
                List<RelNode> subs = new MaterializedViewSubstitutionVisitor(target, root).go(materialization.tableRel);
                for (RelNode s : subs) {
                    call.transformTo(s);
                }
            }
        }
    }
}