Examples with CalciteSemanticException org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException used on opensource projects

Example 6 with CalciteSemanticException

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException in project hive by apache.

the class HiveProject method create.

/**
   * Creates a HiveProject with no sort keys.
   *
   * @param child
   *          input relational expression
   * @param exps
   *          set of expressions for the input columns
   * @param fieldNames
   *          aliases of the expressions
   */
public static HiveProject create(RelNode child, List<? extends RexNode> exps, List<String> fieldNames) throws CalciteSemanticException {
    RelOptCluster cluster = child.getCluster();
    // 1 Ensure columnNames are unique - CALCITE-411
    if (fieldNames != null && !Util.isDistinct(fieldNames)) {
        String msg = "Select list contains multiple expressions with the same name." + fieldNames;
        throw new CalciteSemanticException(msg, UnsupportedFeature.Same_name_in_multiple_expressions);
    }
    RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), exps, fieldNames);
    return create(cluster, child, exps, rowType, Collections.<RelCollation>emptyList());
}

Example 7 with CalciteSemanticException

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException in project hive by apache.

the class CalcitePlanner method genOPTree.

@Override
@SuppressWarnings("rawtypes")
Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
    Operator sinkOp = null;
    boolean skipCalcitePlan = false;
    if (!runCBO) {
        skipCalcitePlan = true;
    } else {
        PreCboCtx cboCtx = (PreCboCtx) plannerCtx;
        // Note: for now, we don't actually pass the queryForCbo to CBO, because
        // it accepts qb, not AST, and can also access all the private stuff in
        // SA. We rely on the fact that CBO ignores the unknown tokens (create
        // table, destination), so if the query is otherwise ok, it is as if we
        // did remove those and gave CBO the proper AST. That is kinda hacky.
        ASTNode queryForCbo = ast;
        if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) {
            // nodeOfInterest is the query
            queryForCbo = cboCtx.nodeOfInterest;
        }
        runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx);
        if (queryProperties.hasMultiDestQuery()) {
            handleMultiDestQuery(ast, cboCtx);
        }
        if (runCBO) {
            profilesCBO = obtainCBOProfiles(queryProperties);
            disableJoinMerge = true;
            boolean reAnalyzeAST = false;
            final boolean materializedView = getQB().isMaterializedView();
            try {
                if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
                    if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) {
                        throw new SemanticException("Create view is not supported in cbo return path.");
                    }
                    sinkOp = getOptimizedHiveOPDag();
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo("Plan optimized by CBO.");
                    this.ctx.setCboSucceeded(true);
                } else {
                    // 1. Gen Optimized AST
                    ASTNode newAST = getOptimizedAST();
                    // 1.1. Fix up the query for insert/ctas/materialized views
                    newAST = fixUpAfterCbo(ast, newAST, cboCtx);
                    // 2. Regen OP plan from optimized AST
                    if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) {
                        try {
                            handleCreateViewDDL(newAST);
                        } catch (SemanticException e) {
                            throw new CalciteViewSemanticException(e.getMessage());
                        }
                    } else {
                        init(false);
                        if (cboCtx.type == PreCboCtx.Type.VIEW && materializedView) {
                            // Redo create-table/view analysis, because it's not part of
                            // doPhase1.
                            // Use the REWRITTEN AST
                            setAST(newAST);
                            newAST = reAnalyzeViewAfterCbo(newAST);
                            // Store text of the ORIGINAL QUERY
                            String originalText = ctx.getTokenRewriteStream().toString(cboCtx.nodeOfInterest.getTokenStartIndex(), cboCtx.nodeOfInterest.getTokenStopIndex());
                            createVwDesc.setViewOriginalText(originalText);
                            viewSelect = newAST;
                            viewsExpanded = new ArrayList<>();
                            viewsExpanded.add(createVwDesc.getViewName());
                        } else if (cboCtx.type == PreCboCtx.Type.CTAS) {
                            // CTAS
                            setAST(newAST);
                            newAST = reAnalyzeCTASAfterCbo(newAST);
                        }
                    }
                    Phase1Ctx ctx_1 = initPhase1Ctx();
                    if (!doPhase1(newAST, getQB(), ctx_1, null)) {
                        throw new RuntimeException("Couldn't do phase1 on CBO optimized query plan");
                    }
                    // unfortunately making prunedPartitions immutable is not possible
                    // here with SemiJoins not all tables are costed in CBO, so their
                    // PartitionList is not evaluated until the run phase.
                    getMetaData(getQB());
                    disableJoinMerge = defaultJoinMerge;
                    sinkOp = genPlan(getQB());
                    LOG.info("CBO Succeeded; optimized logical plan.");
                    this.ctx.setCboInfo("Plan optimized by CBO.");
                    this.ctx.setCboSucceeded(true);
                    if (LOG.isTraceEnabled()) {
                        LOG.trace(newAST.dump());
                    }
                }
            } catch (Exception e) {
                boolean isMissingStats = noColsMissingStats.get() > 0;
                if (isMissingStats) {
                    LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO");
                    this.ctx.setCboInfo("Plan not optimized by CBO due to missing statistics. Please check log for more details.");
                } else {
                    LOG.error("CBO failed, skipping CBO. ", e);
                    if (e instanceof CalciteSemanticException) {
                        CalciteSemanticException calciteSemanticException = (CalciteSemanticException) e;
                        UnsupportedFeature unsupportedFeature = calciteSemanticException.getUnsupportedFeature();
                        if (unsupportedFeature != null) {
                            this.ctx.setCboInfo("Plan not optimized by CBO due to missing feature [" + unsupportedFeature + "].");
                        } else {
                            this.ctx.setCboInfo("Plan not optimized by CBO.");
                        }
                    } else {
                        this.ctx.setCboInfo("Plan not optimized by CBO.");
                    }
                }
                if (e instanceof CalciteSubquerySemanticException) {
                    // so avoid executing subqueries on non-cbo
                    throw new SemanticException(e);
                } else if (e instanceof CalciteViewSemanticException) {
                    // we believe it will throw the same error message
                    throw new SemanticException(e);
                } else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats || e instanceof CalciteSemanticException) {
                    reAnalyzeAST = true;
                } else if (e instanceof SemanticException) {
                    // although, its likely to be a valid exception, we will retry
                    // with cbo off anyway.
                    reAnalyzeAST = true;
                } else if (e instanceof RuntimeException) {
                    throw (RuntimeException) e;
                } else {
                    throw new SemanticException(e);
                }
            } finally {
                runCBO = false;
                disableJoinMerge = defaultJoinMerge;
                disableSemJoinReordering = false;
                if (reAnalyzeAST) {
                    init(true);
                    prunedPartitions.clear();
                    // Assumption: At this point Parse Tree gen & resolution will always
                    // be true (since we started out that way).
                    super.genResolvedParseTree(ast, new PlannerContext());
                    skipCalcitePlan = true;
                }
            }
        } else {
            this.ctx.setCboInfo("Plan not optimized by CBO.");
            skipCalcitePlan = true;
        }
    }
    if (skipCalcitePlan) {
        sinkOp = super.genOPTree(ast, plannerCtx);
    }
    return sinkOp;
}

Example 8 with CalciteSemanticException

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException in project hive by apache.

the class PlanModifierForASTConv method renameTopLevelSelectInResultSchema.

public static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema) throws CalciteSemanticException {
    RelNode parentOforiginalProjRel = topSelparentPair.getKey();
    HiveProject originalProjRel = (HiveProject) topSelparentPair.getValue();
    // Assumption: top portion of tree could only be
    // (limit)?(OB)?(Project)....
    List<RexNode> rootChildExps = originalProjRel.getChildExps();
    if (resultSchema.size() != rootChildExps.size()) {
        // Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now.
        LOG.error(PlanModifierUtil.generateInvalidSchemaMessage(originalProjRel, resultSchema, 0));
        throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
    }
    List<String> newSelAliases = new ArrayList<String>();
    String colAlias;
    for (int i = 0; i < rootChildExps.size(); i++) {
        colAlias = resultSchema.get(i).getName();
        colAlias = getNewColAlias(newSelAliases, colAlias);
        newSelAliases.add(colAlias);
    }
    HiveProject replacementProjectRel = HiveProject.create(originalProjRel.getInput(), originalProjRel.getChildExps(), newSelAliases);
    if (rootRel == originalProjRel) {
        return replacementProjectRel;
    } else {
        parentOforiginalProjRel.replaceInput(0, replacementProjectRel);
        return rootRel;
    }
}

Example 9 with CalciteSemanticException

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException in project hive by apache.

the class HiveMaterializedViewsRegistry method createTableScan.

private static RelNode createTableScan(Table viewTable) {
    // 0. Recreate cluster
    final RelOptPlanner planner = HiveVolcanoPlanner.createPlanner(null);
    final RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl());
    final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
    // 1. Create column schema
    final RowResolver rr = new RowResolver();
    // 1.1 Add Column info for non partion cols (Object Inspector fields)
    StructObjectInspector rowObjectInspector;
    try {
        rowObjectInspector = (StructObjectInspector) viewTable.getDeserializer().getObjectInspector();
    } catch (SerDeException e) {
        // Bail out
        return null;
    }
    List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
    ColumnInfo colInfo;
    String colName;
    ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>();
    for (int i = 0; i < fields.size(); i++) {
        colName = fields.get(i).getFieldName();
        colInfo = new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()), null, false);
        rr.put(null, colName, colInfo);
        cInfoLst.add(colInfo);
    }
    ArrayList<ColumnInfo> nonPartitionColumns = new ArrayList<ColumnInfo>(cInfoLst);
    // 1.2 Add column info corresponding to partition columns
    ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();
    for (FieldSchema part_col : viewTable.getPartCols()) {
        colName = part_col.getName();
        colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true);
        rr.put(null, colName, colInfo);
        cInfoLst.add(colInfo);
        partitionColumns.add(colInfo);
    }
    // 1.3 Build row type from field <type, name>
    RelDataType rowType;
    try {
        rowType = TypeConverter.getType(cluster, rr, null);
    } catch (CalciteSemanticException e) {
        // Bail out
        return null;
    }
    // 2. Build RelOptAbstractTable
    String fullyQualifiedTabName = viewTable.getDbName();
    if (fullyQualifiedTabName != null && !fullyQualifiedTabName.isEmpty()) {
        fullyQualifiedTabName = fullyQualifiedTabName + "." + viewTable.getTableName();
    } else {
        fullyQualifiedTabName = viewTable.getTableName();
    }
    RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<VirtualColumn>(), SessionState.get().getConf(), new HashMap<String, PrunedPartitionList>(), new AtomicInteger());
    RelNode tableRel;
    // 3. Build operator
    if (obtainTableType(viewTable) == TableType.DRUID) {
        // Build Druid query
        String address = HiveConf.getVar(SessionState.get().getConf(), HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
        String dataSource = viewTable.getParameters().get(Constants.DRUID_DATA_SOURCE);
        Set<String> metrics = new HashSet<>();
        List<RelDataType> druidColTypes = new ArrayList<>();
        List<String> druidColNames = new ArrayList<>();
        for (RelDataTypeField field : rowType.getFieldList()) {
            druidColTypes.add(field.getType());
            druidColNames.add(field.getName());
            if (field.getName().equals(DruidTable.DEFAULT_TIMESTAMP_COLUMN)) {
                // timestamp
                continue;
            }
            if (field.getType().getSqlTypeName() == SqlTypeName.VARCHAR) {
                // dimension
                continue;
            }
            metrics.add(field.getName());
        }
        List<Interval> intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
        DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals);
        final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false);
        tableRel = DruidQuery.create(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, druidTable, ImmutableList.<RelNode>of(scan));
    } else {
        // Build Hive Table Scan Rel
        tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false);
    }
    return tableRel;
}

Example 10 with CalciteSemanticException

use of org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException in project hive by apache.

the class SqlFunctionConverter method getCalciteFn.

public static SqlOperator getCalciteFn(String hiveUdfName, ImmutableList<RelDataType> calciteArgTypes, RelDataType calciteRetType, boolean deterministic) throws CalciteSemanticException {
    if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) {
        // this.So, bail out for now.
        throw new CalciteSemanticException("<=> is not yet supported for cbo.", UnsupportedFeature.Less_than_equal_greater_than);
    }
    SqlOperator calciteOp;
    CalciteUDFInfo uInf = getUDFInfo(hiveUdfName, calciteArgTypes, calciteRetType);
    switch(hiveUdfName) {
        //TODO: Perhaps we should do this for all functions, not just +,-
        case "-":
            calciteOp = new SqlMonotonicBinaryOperator("-", SqlKind.MINUS, 40, true, uInf.returnTypeInference, uInf.operandTypeInference, OperandTypes.MINUS_OPERATOR);
            break;
        case "+":
            calciteOp = new SqlMonotonicBinaryOperator("+", SqlKind.PLUS, 40, true, uInf.returnTypeInference, uInf.operandTypeInference, OperandTypes.PLUS_OPERATOR);
            break;
        default:
            calciteOp = hiveToCalcite.get(hiveUdfName);
            if (null == calciteOp) {
                calciteOp = new CalciteSqlFn(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, uInf.operandTypeInference, uInf.operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION, deterministic);
            }
            break;
    }
    return calciteOp;
}