Examples with ColumnInfo org.apache.hadoop.hive.ql.exec.ColumnInfo used on opensource projects

Example 81 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class RowResolver method add.

// TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or not?
private static boolean add(RowResolver rrToAddTo, RowResolver rrToAddFrom, IntRef outputColPosRef, int numColumns) throws SemanticException {
    boolean hasDuplicates = false;
    String tabAlias;
    String colAlias;
    String[] qualifiedColName;
    int i = 0;
    int outputColPos = outputColPosRef == null ? 0 : outputColPosRef.val;
    for (ColumnInfo cInfoFrmInput : rrToAddFrom.getRowSchema().getSignature()) {
        if (numColumns >= 0 && i == numColumns) {
            break;
        }
        ColumnInfo newCI = null;
        String internalName = cInfoFrmInput.getInternalName();
        qualifiedColName = rrToAddFrom.reverseLookup(internalName);
        tabAlias = qualifiedColName[0];
        colAlias = qualifiedColName[1];
        newCI = new ColumnInfo(cInfoFrmInput);
        newCI.setInternalName(SemanticAnalyzer.getColumnInternalName(outputColPos));
        outputColPos++;
        boolean isUnique = rrToAddTo.putWithCheck(tabAlias, colAlias, internalName, newCI);
        hasDuplicates |= (!isUnique);
        qualifiedColName = rrToAddFrom.getAlternateMappings(internalName);
        if (qualifiedColName != null) {
            tabAlias = qualifiedColName[0];
            colAlias = qualifiedColName[1];
            rrToAddTo.put(tabAlias, colAlias, newCI);
        }
        i++;
    }
    if (outputColPosRef != null) {
        outputColPosRef.val = outputColPos;
    }
    return !hasDuplicates;
}

Example 82 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class PTFTranslator method buildRowResolverForWindowing.

protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def) throws SemanticException {
    RowResolver rr = new RowResolver();
    HashMap<String, WindowExpressionSpec> aliasToExprMap = windowingSpec.getAliasToWdwExpr();
    /*
     * add Window Functions
     */
    for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
        ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression();
        ObjectInspector wFnOI = null;
        if (wFnDef.isPivotResult()) {
            wFnOI = ((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector();
        } else {
            wFnOI = wFnDef.getOI();
        }
        ColumnInfo cInfo = new ColumnInfo(wFnDef.getAlias(), TypeInfoUtils.getTypeInfoFromObjectInspector(wFnOI), null, true, true);
        rr.putExpression(ast, cInfo);
    }
    RowResolver inpRR = def.getRawInputShape().getRr();
    /*
     * add columns from inpRR
     */
    for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
        ColumnInfo cInfo = new ColumnInfo(inpCInfo);
        ASTNode inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR);
        if (inExpr != null) {
            rr.putExpression(inExpr, cInfo);
        } else {
            String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
            if (tabColAlias != null) {
                rr.put(tabColAlias[0], tabColAlias[1], cInfo);
            } else {
                rr.put(inpCInfo.getTabAlias(), inpCInfo.getAlias(), cInfo);
            }
        }
        String[] altMapping = inputRR.getAlternateMappings(inpCInfo.getInternalName());
        if (altMapping != null) {
            rr.put(altMapping[0], altMapping[1], cInfo);
        }
    }
    return rr;
}

Example 83 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class QBSubQuery method buildJoinCondition.

void buildJoinCondition(RowResolver outerQueryRR, RowResolver sqRR, boolean forHavingClause, String outerQueryAlias) throws SemanticException {
    ASTNode parentQueryJoinCond = null;
    if (parentQueryExpression != null) {
        ColumnInfo outerQueryCol = null;
        try {
            outerQueryCol = outerQueryRR.getExpression(parentQueryExpression);
        } catch (SemanticException se) {
        // ignore
        }
        ASTNode parentExpr = parentQueryExpression;
        if (!forHavingClause) {
            Set<String> aliases = outerQueryRR.getRslvMap().keySet();
            if (notInCheck != null) {
                aliases.remove(notInCheck.getAlias());
            }
            String tableAlias = aliases.size() == 1 ? aliases.iterator().next() : null;
            parentExpr = SubQueryUtils.setQualifiedColumnReferences(parentExpr, tableAlias);
            if (parentExpr == null) {
                subQueryAST.setOrigin(originalSQASTOrigin);
                throw new SemanticException(ASTErrorUtils.getMsg(ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg(), parentQueryExpression, "Correlating expression contains ambiguous column references."));
            }
        }
        parentQueryJoinCond = SubQueryUtils.buildOuterQryToSQJoinCond(parentExpr, alias, sqRR);
        if (outerQueryCol != null) {
            rewriteCorrConjunctForHaving(parentQueryJoinCond, true, outerQueryAlias, outerQueryRR, outerQueryCol);
        }
        subQueryDiagnostic.addJoinCondition(parentQueryJoinCond, outerQueryCol != null, true);
    }
    joinConditionAST = SubQueryUtils.andAST(parentQueryJoinCond, joinConditionAST);
    setJoinType();
    if (joinType == JoinType.LEFTOUTER) {
        if (operator.getType() == SubQueryType.NOT_EXISTS && hasCorrelation) {
            postJoinConditionAST = SubQueryUtils.buildPostJoinNullCheck(subQueryJoinAliasExprs);
        } else if (operator.getType() == SubQueryType.NOT_IN) {
            postJoinConditionAST = SubQueryUtils.buildOuterJoinPostCond(alias, sqRR);
        }
    }
}

Example 84 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class ExprNodeDescUtils method genExprNodeDesc.

/**
 * Build ExprNodeColumnDesc for the projections in the input operator from
 * sartpos to endpos(both included). Operator must have an associated
 * colExprMap.
 *
 * @param inputOp
 *          Input Hive Operator
 * @param startPos
 *          starting position in the input operator schema; must be >=0 and <=
 *          endPos
 * @param endPos
 *          end position in the input operator schema; must be >=0.
 * @return List of ExprNodeDesc
 */
public static ArrayList<ExprNodeDesc> genExprNodeDesc(Operator inputOp, int startPos, int endPos, boolean addEmptyTabAlias, boolean setColToNonVirtual) {
    ArrayList<ExprNodeDesc> exprColLst = new ArrayList<ExprNodeDesc>();
    List<ColumnInfo> colInfoLst = inputOp.getSchema().getSignature();
    String tabAlias;
    boolean vc;
    ColumnInfo ci;
    for (int i = startPos; i <= endPos; i++) {
        ci = colInfoLst.get(i);
        tabAlias = ci.getTabAlias();
        if (addEmptyTabAlias) {
            tabAlias = "";
        }
        vc = ci.getIsVirtualCol();
        if (setColToNonVirtual) {
            vc = false;
        }
        exprColLst.add(new ExprNodeColumnDesc(ci.getType(), ci.getInternalName(), tabAlias, vc));
    }
    return exprColLst;
}

Example 85 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class HiveSortLimitVisitor method visit.

@Override
OpAttr visit(HiveSortLimit sortRel) throws SemanticException {
    OpAttr inputOpAf = hiveOpConverter.dispatch(sortRel.getInput());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + sortRel.getId() + ":" + sortRel.getRelTypeName() + " with row type: [" + sortRel.getRowType() + "]");
        if (sortRel.getCollation() == RelCollations.EMPTY) {
            LOG.debug("Operator rel#" + sortRel.getId() + ":" + sortRel.getRelTypeName() + " consists of limit");
        } else if (sortRel.fetch == null) {
            LOG.debug("Operator rel#" + sortRel.getId() + ":" + sortRel.getRelTypeName() + " consists of sort");
        } else {
            LOG.debug("Operator rel#" + sortRel.getId() + ":" + sortRel.getRelTypeName() + " consists of sort+limit");
        }
    }
    Operator<?> inputOp = inputOpAf.inputs.get(0);
    Operator<?> resultOp = inputOpAf.inputs.get(0);
    // of their columns
    if (sortRel.getCollation() != RelCollations.EMPTY) {
        // In strict mode, in the presence of order by, limit must be specified.
        if (sortRel.fetch == null) {
            String error = StrictChecks.checkNoLimit(hiveOpConverter.getHiveConf());
            if (error != null)
                throw new SemanticException(error);
        }
        // 1.a. Extract order for each column from collation
        // Generate sortCols and order
        ImmutableBitSet.Builder sortColsPosBuilder = ImmutableBitSet.builder();
        ImmutableBitSet.Builder sortOutputColsPosBuilder = ImmutableBitSet.builder();
        Map<Integer, RexNode> obRefToCallMap = sortRel.getInputRefToCallMap();
        List<ExprNodeDesc> sortCols = new ArrayList<ExprNodeDesc>();
        StringBuilder order = new StringBuilder();
        StringBuilder nullOrder = new StringBuilder();
        for (RelFieldCollation sortInfo : sortRel.getCollation().getFieldCollations()) {
            int sortColumnPos = sortInfo.getFieldIndex();
            ColumnInfo columnInfo = new ColumnInfo(inputOp.getSchema().getSignature().get(sortColumnPos));
            ExprNodeColumnDesc sortColumn = new ExprNodeColumnDesc(columnInfo.getType(), columnInfo.getInternalName(), columnInfo.getTabAlias(), columnInfo.getIsVirtualCol());
            sortCols.add(sortColumn);
            if (sortInfo.getDirection() == RelFieldCollation.Direction.DESCENDING) {
                order.append("-");
            } else {
                order.append("+");
            }
            if (sortInfo.nullDirection == RelFieldCollation.NullDirection.FIRST) {
                nullOrder.append("a");
            } else if (sortInfo.nullDirection == RelFieldCollation.NullDirection.LAST) {
                nullOrder.append("z");
            } else {
                // Default
                nullOrder.append(sortInfo.getDirection() == RelFieldCollation.Direction.DESCENDING ? "z" : "a");
            }
            if (obRefToCallMap != null) {
                RexNode obExpr = obRefToCallMap.get(sortColumnPos);
                sortColsPosBuilder.set(sortColumnPos);
                if (obExpr == null) {
                    sortOutputColsPosBuilder.set(sortColumnPos);
                }
            }
        }
        // Use only 1 reducer for order by
        int numReducers = 1;
        // We keep the columns only the columns that are part of the final output
        List<String> keepColumns = new ArrayList<String>();
        final ImmutableBitSet sortColsPos = sortColsPosBuilder.build();
        final ImmutableBitSet sortOutputColsPos = sortOutputColsPosBuilder.build();
        final List<ColumnInfo> inputSchema = inputOp.getSchema().getSignature();
        for (int pos = 0; pos < inputSchema.size(); pos++) {
            if ((sortColsPos.get(pos) && sortOutputColsPos.get(pos)) || (!sortColsPos.get(pos) && !sortOutputColsPos.get(pos))) {
                keepColumns.add(inputSchema.get(pos).getInternalName());
            }
        }
        // 1.b. Generate reduce sink and project operator
        resultOp = HiveOpConverterUtils.genReduceSinkAndBacktrackSelect(resultOp, sortCols.toArray(new ExprNodeDesc[sortCols.size()]), 0, new ArrayList<ExprNodeDesc>(), order.toString(), nullOrder.toString(), numReducers, Operation.NOT_ACID, hiveOpConverter.getHiveConf(), keepColumns);
    }
    // 2. If we need to generate limit
    if (sortRel.fetch != null) {
        int limit = RexLiteral.intValue(sortRel.fetch);
        int offset = sortRel.offset == null ? 0 : RexLiteral.intValue(sortRel.offset);
        LimitDesc limitDesc = new LimitDesc(offset, limit);
        ArrayList<ColumnInfo> cinfoLst = HiveOpConverterUtils.createColInfos(resultOp);
        resultOp = OperatorFactory.getAndMakeChild(limitDesc, new RowSchema(cinfoLst), resultOp);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Generated " + resultOp + " with row schema: [" + resultOp.getSchema() + "]");
        }
    }
    // 3. Return result
    return inputOpAf.clone(resultOp);
}