Examples with UnsupportedFeature org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature used on opensource projects

Example 1 with UnsupportedFeature

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature in project hive by apache.

the class CalcitePlanner method genOPTree.

@Override
@SuppressWarnings("rawtypes")
Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
    final Operator sinkOp;
    if (!runCBO) {
        sinkOp = super.genOPTree(ast, plannerCtx);
    } else {
        PreCboCtx cboCtx = (PreCboCtx) plannerCtx;
        List<ASTNode> oldHints = new ArrayList<>();
        // Cache the hints before CBO runs and removes them.
        // Use the hints later in top level QB.
        getHintsFromQB(getQB(), oldHints);
        // Note: for now, we don't actually pass the queryForCbo to CBO, because
        // it accepts qb, not AST, and can also access all the private stuff in
        // SA. We rely on the fact that CBO ignores the unknown tokens (create
        // table, destination), so if the query is otherwise ok, it is as if we
        // did remove those and gave CBO the proper AST. That is kinda hacky.
        ASTNode queryForCbo = ast;
        if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) {
            // nodeOfInterest is the query
            queryForCbo = cboCtx.nodeOfInterest;
        }
        Pair<Boolean, String> canCBOHandleReason = canCBOHandleAst(queryForCbo, getQB(), cboCtx);
        runCBO = canCBOHandleReason.left;
        if (queryProperties.hasMultiDestQuery()) {
            handleMultiDestQuery(ast, cboCtx);
        }
        if (runCBO) {
            profilesCBO = obtainCBOProfiles(queryProperties);
            disableJoinMerge = true;
            final boolean materializedView = getQB().isMaterializedView();
            try {
                // 0. Gen Optimized Plan
                RelNode newPlan = logicalPlan();
                if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
                    if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) {
                        throw new SemanticException("Create view is not supported in cbo return path.");
                    }
                    sinkOp = getOptimizedHiveOPDag(newPlan);
                    if (oldHints.size() > 0) {
                        LOG.debug("Propagating hints to QB: " + oldHints);
                        getQB().getParseInfo().setHintList(oldHints);
                    }
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo(getOptimizedByCboInfo());
                    this.ctx.setCboSucceeded(true);
                } else {
                    // 1. Convert Plan to AST
                    ASTNode newAST = getOptimizedAST(newPlan);
                    // 1.1. Fix up the query for insert/ctas/materialized views
                    newAST = fixUpAfterCbo(ast, newAST, cboCtx);
                    // 2. Regen OP plan from optimized AST
                    if (forViewCreation) {
                        // the reset would remove the translations
                        executeUnparseTranlations();
                        // save the resultSchema before rewriting it
                        originalResultSchema = resultSchema;
                    }
                    if (cboCtx.type == PreCboCtx.Type.VIEW) {
                        try {
                            viewSelect = handleCreateViewDDL(newAST);
                        } catch (SemanticException e) {
                            throw new CalciteViewSemanticException(e.getMessage());
                        }
                    } else if (cboCtx.type == PreCboCtx.Type.CTAS) {
                        // CTAS
                        init(false);
                        setAST(newAST);
                        newAST = reAnalyzeCTASAfterCbo(newAST);
                    } else {
                        // All others
                        init(false);
                    }
                    if (oldHints.size() > 0) {
                        if (getQB().getParseInfo().getHints() != null) {
                            LOG.warn("Hints are not null in the optimized tree; " + "after CBO " + getQB().getParseInfo().getHints().dump());
                        } else {
                            LOG.debug("Propagating hints to QB: " + oldHints);
                            getQB().getParseInfo().setHintList(oldHints);
                        }
                    }
                    Phase1Ctx ctx_1 = initPhase1Ctx();
                    if (!doPhase1(newAST, getQB(), ctx_1, null)) {
                        throw new RuntimeException("Couldn't do phase1 on CBO optimized query plan");
                    }
                    // unfortunately making prunedPartitions immutable is not possible
                    // here with SemiJoins not all tables are costed in CBO, so their
                    // PartitionList is not evaluated until the run phase.
                    getMetaData(getQB());
                    disableJoinMerge = defaultJoinMerge;
                    sinkOp = genPlan(getQB());
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo(getOptimizedByCboInfo());
                    this.ctx.setCboSucceeded(true);
                    if (this.ctx.isExplainPlan()) {
                        // Enrich explain with information derived from CBO
                        ExplainConfiguration explainConfig = this.ctx.getExplainConfig();
                        if (explainConfig.isCbo()) {
                            if (!explainConfig.isCboJoinCost()) {
                                // Include cost as provided by Calcite
                                newPlan.getCluster().invalidateMetadataQuery();
                                RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.DEFAULT);
                            }
                            if (explainConfig.isFormatted()) {
                                this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan));
                            } else if (explainConfig.isCboCost() || explainConfig.isCboJoinCost()) {
                                this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan, SqlExplainLevel.ALL_ATTRIBUTES));
                            } else {
                                // Do not include join cost
                                this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan));
                            }
                        } else if (explainConfig.isFormatted()) {
                            this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan));
                            this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
                        } else if (explainConfig.isExtended()) {
                            this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
                        }
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace(getOptimizedSql(newPlan));
                        LOG.trace(newAST.dump());
                    }
                }
            } catch (Exception e) {
                LOG.error("CBO failed, skipping CBO. ", e);
                String cboMsg = "Plan not optimized by CBO.";
                boolean isMissingStats = noColsMissingStats.get() > 0;
                if (isMissingStats) {
                    LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO");
                    cboMsg = "Plan not optimized by CBO due to missing statistics. Please check log for more details.";
                } else if (e instanceof CalciteSemanticException) {
                    CalciteSemanticException cse = (CalciteSemanticException) e;
                    UnsupportedFeature unsupportedFeature = cse.getUnsupportedFeature();
                    if (unsupportedFeature != null) {
                        cboMsg = "Plan not optimized by CBO due to missing feature [" + unsupportedFeature + "].";
                    }
                }
                this.ctx.setCboInfo(cboMsg);
                // Determine if we should re-throw the exception OR if we try to mark the query to retry as non-CBO.
                if (fallbackStrategy.isFatal(e)) {
                    if (e instanceof RuntimeException || e instanceof SemanticException) {
                        // These types of exceptions do not need wrapped
                        throw e;
                    }
                    // Wrap all other errors (Should only hit in tests)
                    throw new SemanticException(e);
                } else {
                    throw new ReCompileException(this.ctx.getCboInfo());
                }
            } finally {
                runCBO = false;
                disableJoinMerge = defaultJoinMerge;
                disableSemJoinReordering = false;
            }
        } else {
            String msg;
            if (canCBOHandleReason.right != null) {
                msg = "Plan not optimized by CBO because the statement " + canCBOHandleReason.right;
            } else {
                msg = "Plan not optimized by CBO.";
            }
            this.ctx.setCboInfo(msg);
            sinkOp = super.genOPTree(ast, plannerCtx);
        }
    }
    return sinkOp;
}