Examples with ReCompileException org.apache.hadoop.hive.ql.reexec.ReCompileException used on opensource projects

Example 1 with ReCompileException

use of org.apache.hadoop.hive.ql.reexec.ReCompileException in project hive by apache.

the class CalcitePlanner method genOPTree.

@Override
@SuppressWarnings("rawtypes")
Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
    final Operator sinkOp;
    if (!runCBO) {
        sinkOp = super.genOPTree(ast, plannerCtx);
    } else {
        PreCboCtx cboCtx = (PreCboCtx) plannerCtx;
        List<ASTNode> oldHints = new ArrayList<>();
        // Cache the hints before CBO runs and removes them.
        // Use the hints later in top level QB.
        getHintsFromQB(getQB(), oldHints);
        // Note: for now, we don't actually pass the queryForCbo to CBO, because
        // it accepts qb, not AST, and can also access all the private stuff in
        // SA. We rely on the fact that CBO ignores the unknown tokens (create
        // table, destination), so if the query is otherwise ok, it is as if we
        // did remove those and gave CBO the proper AST. That is kinda hacky.
        ASTNode queryForCbo = ast;
        if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) {
            // nodeOfInterest is the query
            queryForCbo = cboCtx.nodeOfInterest;
        }
        Pair<Boolean, String> canCBOHandleReason = canCBOHandleAst(queryForCbo, getQB(), cboCtx);
        runCBO = canCBOHandleReason.left;
        if (queryProperties.hasMultiDestQuery()) {
            handleMultiDestQuery(ast, cboCtx);
        }
        if (runCBO) {
            profilesCBO = obtainCBOProfiles(queryProperties);
            disableJoinMerge = true;
            final boolean materializedView = getQB().isMaterializedView();
            try {
                // 0. Gen Optimized Plan
                RelNode newPlan = logicalPlan();
                if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
                    if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) {
                        throw new SemanticException("Create view is not supported in cbo return path.");
                    }
                    sinkOp = getOptimizedHiveOPDag(newPlan);
                    if (oldHints.size() > 0) {
                        LOG.debug("Propagating hints to QB: " + oldHints);
                        getQB().getParseInfo().setHintList(oldHints);
                    }
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo(getOptimizedByCboInfo());
                    this.ctx.setCboSucceeded(true);
                } else {
                    // 1. Convert Plan to AST
                    ASTNode newAST = getOptimizedAST(newPlan);
                    // 1.1. Fix up the query for insert/ctas/materialized views
                    newAST = fixUpAfterCbo(ast, newAST, cboCtx);
                    // 2. Regen OP plan from optimized AST
                    if (forViewCreation) {
                        // the reset would remove the translations
                        executeUnparseTranlations();
                        // save the resultSchema before rewriting it
                        originalResultSchema = resultSchema;
                    }
                    if (cboCtx.type == PreCboCtx.Type.VIEW) {
                        try {
                            viewSelect = handleCreateViewDDL(newAST);
                        } catch (SemanticException e) {
                            throw new CalciteViewSemanticException(e.getMessage());
                        }
                    } else if (cboCtx.type == PreCboCtx.Type.CTAS) {
                        // CTAS
                        init(false);
                        setAST(newAST);
                        newAST = reAnalyzeCTASAfterCbo(newAST);
                    } else {
                        // All others
                        init(false);
                    }
                    if (oldHints.size() > 0) {
                        if (getQB().getParseInfo().getHints() != null) {
                            LOG.warn("Hints are not null in the optimized tree; " + "after CBO " + getQB().getParseInfo().getHints().dump());
                        } else {
                            LOG.debug("Propagating hints to QB: " + oldHints);
                            getQB().getParseInfo().setHintList(oldHints);
                        }
                    }
                    Phase1Ctx ctx_1 = initPhase1Ctx();
                    if (!doPhase1(newAST, getQB(), ctx_1, null)) {
                        throw new RuntimeException("Couldn't do phase1 on CBO optimized query plan");
                    }
                    // unfortunately making prunedPartitions immutable is not possible
                    // here with SemiJoins not all tables are costed in CBO, so their
                    // PartitionList is not evaluated until the run phase.
                    getMetaData(getQB());
                    disableJoinMerge = defaultJoinMerge;
                    sinkOp = genPlan(getQB());
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo(getOptimizedByCboInfo());
                    this.ctx.setCboSucceeded(true);
                    if (this.ctx.isExplainPlan()) {
                        // Enrich explain with information derived from CBO
                        ExplainConfiguration explainConfig = this.ctx.getExplainConfig();
                        if (explainConfig.isCbo()) {
                            if (!explainConfig.isCboJoinCost()) {
                                // Include cost as provided by Calcite
                                newPlan.getCluster().invalidateMetadataQuery();
                                RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.DEFAULT);
                            }
                            if (explainConfig.isFormatted()) {
                                this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan));
                            } else if (explainConfig.isCboCost() || explainConfig.isCboJoinCost()) {
                                this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan, SqlExplainLevel.ALL_ATTRIBUTES));
                            } else {
                                // Do not include join cost
                                this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan));
                            }
                        } else if (explainConfig.isFormatted()) {
                            this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan));
                            this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
                        } else if (explainConfig.isExtended()) {
                            this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
                        }
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace(getOptimizedSql(newPlan));
                        LOG.trace(newAST.dump());
                    }
                }
            } catch (Exception e) {
                LOG.error("CBO failed, skipping CBO. ", e);
                String cboMsg = "Plan not optimized by CBO.";
                boolean isMissingStats = noColsMissingStats.get() > 0;
                if (isMissingStats) {
                    LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO");
                    cboMsg = "Plan not optimized by CBO due to missing statistics. Please check log for more details.";
                } else if (e instanceof CalciteSemanticException) {
                    CalciteSemanticException cse = (CalciteSemanticException) e;
                    UnsupportedFeature unsupportedFeature = cse.getUnsupportedFeature();
                    if (unsupportedFeature != null) {
                        cboMsg = "Plan not optimized by CBO due to missing feature [" + unsupportedFeature + "].";
                    }
                }
                this.ctx.setCboInfo(cboMsg);
                // Determine if we should re-throw the exception OR if we try to mark the query to retry as non-CBO.
                if (fallbackStrategy.isFatal(e)) {
                    if (e instanceof RuntimeException || e instanceof SemanticException) {
                        // These types of exceptions do not need wrapped
                        throw e;
                    }
                    // Wrap all other errors (Should only hit in tests)
                    throw new SemanticException(e);
                } else {
                    throw new ReCompileException(this.ctx.getCboInfo());
                }
            } finally {
                runCBO = false;
                disableJoinMerge = defaultJoinMerge;
                disableSemJoinReordering = false;
            }
        } else {
            String msg;
            if (canCBOHandleReason.right != null) {
                msg = "Plan not optimized by CBO because the statement " + canCBOHandleReason.right;
            } else {
                msg = "Plan not optimized by CBO.";
            }
            this.ctx.setCboInfo(msg);
            sinkOp = super.genOPTree(ast, plannerCtx);
        }
    }
    return sinkOp;
}

Example 2 with ReCompileException

use of org.apache.hadoop.hive.ql.reexec.ReCompileException in project hive by apache.

the class ExplainSemanticAnalyzer method analyzeInternal.

@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
    final int childCount = ast.getChildCount();
    // Skip TOK_QUERY.
    int i = 1;
    while (i < childCount) {
        int explainOptions = ast.getChild(i).getType();
        if (explainOptions == HiveParser.KW_FORMATTED) {
            config.setFormatted(true);
        } else if (explainOptions == HiveParser.KW_EXTENDED) {
            config.setExtended(true);
        } else if (explainOptions == HiveParser.KW_DEPENDENCY) {
            config.setDependency(true);
        } else if (explainOptions == HiveParser.KW_CBO) {
            config.setCbo(true);
        } else if (explainOptions == HiveParser.KW_COST) {
            config.setCboCost(true);
        } else if (explainOptions == HiveParser.KW_JOINCOST) {
            config.setCboJoinCost(true);
        } else if (explainOptions == HiveParser.KW_LOGICAL) {
            config.setLogical(true);
        } else if (explainOptions == HiveParser.KW_AUTHORIZATION) {
            config.setAuthorize(true);
        } else if (explainOptions == HiveParser.KW_ANALYZE) {
            config.setAnalyze(AnalyzeState.RUNNING);
            config.setExplainRootPath(ctx.getMRTmpPath());
        } else if (explainOptions == HiveParser.KW_VECTORIZATION) {
            config.setVectorization(true);
            if (i + 1 < childCount) {
                int vectorizationOption = ast.getChild(i + 1).getType();
                // [ONLY]
                if (vectorizationOption == HiveParser.TOK_ONLY) {
                    config.setVectorizationOnly(true);
                    i++;
                    if (i + 1 >= childCount) {
                        break;
                    }
                    vectorizationOption = ast.getChild(i + 1).getType();
                }
                // [SUMMARY|OPERATOR|EXPRESSION|DETAIL]
                if (vectorizationOption == HiveParser.TOK_SUMMARY) {
                    config.setVectorizationDetailLevel(VectorizationDetailLevel.SUMMARY);
                    i++;
                } else if (vectorizationOption == HiveParser.TOK_OPERATOR) {
                    config.setVectorizationDetailLevel(VectorizationDetailLevel.OPERATOR);
                    i++;
                } else if (vectorizationOption == HiveParser.TOK_EXPRESSION) {
                    config.setVectorizationDetailLevel(VectorizationDetailLevel.EXPRESSION);
                    i++;
                } else if (vectorizationOption == HiveParser.TOK_DETAIL) {
                    config.setVectorizationDetailLevel(VectorizationDetailLevel.DETAIL);
                    i++;
                }
            }
        } else if (explainOptions == HiveParser.KW_LOCKS) {
            config.setLocks(true);
        } else if (explainOptions == HiveParser.KW_AST) {
            config.setAst(true);
        } else if (explainOptions == HiveParser.KW_DEBUG) {
            config.setDebug(true);
        } else if (explainOptions == HiveParser.KW_DDL) {
            config.setDDL(true);
            config.setCbo(true);
            config.setVectorization(true);
        } else {
        // UNDONE: UNKNOWN OPTION?
        }
        i++;
    }
    ctx.setExplainConfig(config);
    ctx.setExplainPlan(true);
    ASTNode input = (ASTNode) ast.getChild(0);
    // step 2 (ANALYZE_STATE.ANALYZING), explain the query and provide the runtime #rows collected.
    if (config.getAnalyze() == AnalyzeState.RUNNING) {
        String query = ctx.getTokenRewriteStream().toString(input.getTokenStartIndex(), input.getTokenStopIndex());
        LOG.info("Explain analyze (running phase) for query " + query);
        conf.unset(ValidTxnList.VALID_TXNS_KEY);
        conf.unset(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
        Context runCtx = null;
        try {
            runCtx = new Context(conf);
            // runCtx and ctx share the configuration, but not isExplainPlan()
            runCtx.setExplainConfig(config);
            try (Driver driver = new Driver(conf, runCtx, queryState.getLineageState())) {
                driver.run(query);
                while (driver.getResults(new ArrayList<String>())) {
                }
            } catch (CommandProcessorException e) {
                if (e.getCause() instanceof ReCompileException) {
                    throw (ReCompileException) e.getCause();
                } else {
                    throw new SemanticException(e.getMessage(), e);
                }
            }
            config.setOpIdToRuntimeNumRows(aggregateStats(config.getExplainRootPath()));
        } catch (IOException e1) {
            throw new SemanticException(e1);
        }
        ctx.resetOpContext();
        ctx.resetStream();
        TaskFactory.resetId();
        LOG.info("Explain analyze (analyzing phase) for query " + query);
        config.setAnalyze(AnalyzeState.ANALYZING);
    }
    // Creating new QueryState unfortunately causes all .q.out to change - do this in a separate ticket
    // Sharing QueryState between generating the plan and executing the query seems bad
    // BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(new QueryState(queryState.getConf()), input);
    BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, input);
    sem.analyze(input, ctx);
    sem.validate();
    inputs = sem.getInputs();
    outputs = sem.getOutputs();
    ctx.setResFile(ctx.getLocalTmpPath());
    List<Task<?>> tasks = sem.getAllRootTasks();
    if (tasks == null) {
        tasks = Collections.emptyList();
    }
    FetchTask fetchTask = sem.getFetchTask();
    if (fetchTask != null) {
        // Initialize fetch work such that operator tree will be constructed.
        fetchTask.getWork().initializeForFetch(ctx.getOpContext());
    }
    if (sem instanceof SemanticAnalyzer) {
        pCtx = sem.getParseContext();
    }
    config.setUserLevelExplain(!config.isExtended() && !config.isFormatted() && !config.isDependency() && !config.isCbo() && !config.isLogical() && !config.isVectorization() && !config.isAuthorize() && ((HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_EXPLAIN_USER) && HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) || (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_SPARK_EXPLAIN_USER) && HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark"))));
    ExplainWork work = new ExplainWork(ctx.getResFile(), pCtx, tasks, fetchTask, input, sem, config, ctx.getCboInfo(), ctx.getOptimizedSql(), ctx.getCalcitePlan());
    work.setAppendTaskType(HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES));
    ExplainTask explTask = (ExplainTask) TaskFactory.get(work);
    fieldList = ExplainTask.getResultSchema();
    rootTasks.add(explTask);
}