Examples with ColumnInfo org.apache.hadoop.hive.ql.exec.ColumnInfo used on opensource projects

Example 51 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class RelOptHiveTable method copy.

public RelOptHiveTable copy(RelDataType newRowType) {
    // 1. Build map of column name to col index of original schema
    // Assumption: Hive Table can not contain duplicate column names
    Map<String, Integer> nameToColIndxMap = new HashMap<String, Integer>();
    for (RelDataTypeField f : this.rowType.getFieldList()) {
        nameToColIndxMap.put(f.getName(), f.getIndex());
    }
    // 2. Build nonPart/Part/Virtual column info for new RowSchema
    List<ColumnInfo> newHiveNonPartitionCols = new ArrayList<ColumnInfo>();
    List<ColumnInfo> newHivePartitionCols = new ArrayList<ColumnInfo>();
    List<VirtualColumn> newHiveVirtualCols = new ArrayList<VirtualColumn>();
    Map<Integer, VirtualColumn> virtualColInfoMap = HiveCalciteUtil.getVColsMap(this.hiveVirtualCols, this.noOfNonVirtualCols);
    Integer originalColIndx;
    ColumnInfo cInfo;
    VirtualColumn vc;
    for (RelDataTypeField f : newRowType.getFieldList()) {
        originalColIndx = nameToColIndxMap.get(f.getName());
        if ((cInfo = hiveNonPartitionColsMap.get(originalColIndx)) != null) {
            newHiveNonPartitionCols.add(new ColumnInfo(cInfo));
        } else if ((cInfo = hivePartitionColsMap.get(originalColIndx)) != null) {
            newHivePartitionCols.add(new ColumnInfo(cInfo));
        } else if ((vc = virtualColInfoMap.get(originalColIndx)) != null) {
            newHiveVirtualCols.add(vc);
        } else {
            throw new RuntimeException("Copy encountered a column not seen in original TS");
        }
    }
    // 3. Build new Table
    return new RelOptHiveTable(this.schema, this.typeFactory, this.qualifiedTblName, newRowType, this.hiveTblMetadata, newHiveNonPartitionCols, newHivePartitionCols, newHiveVirtualCols, this.hiveConf, this.db, this.tablesCache, this.partitionCache, this.colStatsCache, this.noColsMissingStats);
}

Example 52 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class HiveFilterVisitor method visit.

/**
 * TODO: 1) isSamplingPred 2) sampleDesc 3) isSortedFilter.
 */
@Override
OpAttr visit(HiveFilter filterRel) throws SemanticException {
    OpAttr inputOpAf = hiveOpConverter.dispatch(filterRel.getInput());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + filterRel.getId() + ":" + filterRel.getRelTypeName() + " with row type: [" + filterRel.getRowType() + "]");
    }
    ExprNodeDesc filCondExpr = filterRel.getCondition().accept(new ExprNodeConverter(inputOpAf.tabAlias, filterRel.getInput().getRowType(), inputOpAf.vcolsInCalcite, filterRel.getCluster().getTypeFactory(), true));
    FilterDesc filDesc = new FilterDesc(filCondExpr, false);
    ArrayList<ColumnInfo> cinfoLst = HiveOpConverterUtils.createColInfos(inputOpAf.inputs.get(0));
    FilterOperator filOp = (FilterOperator) OperatorFactory.getAndMakeChild(filDesc, new RowSchema(cinfoLst), inputOpAf.inputs.get(0));
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + filOp + " with row schema: [" + filOp.getSchema() + "]");
    }
    return inputOpAf.clone(filOp);
}

Example 53 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class HiveProjectVisitor method createColInfos.

private Pair<ArrayList<ColumnInfo>, Set<Integer>> createColInfos(List<RexNode> calciteExprs, List<ExprNodeDesc> hiveExprs, List<String> projNames, OpAttr inpOpAf) {
    if (hiveExprs.size() != projNames.size()) {
        throw new RuntimeException("Column expressions list doesn't match Column Names list");
    }
    RexNode rexN;
    ExprNodeDesc pe;
    ArrayList<ColumnInfo> colInfos = new ArrayList<ColumnInfo>();
    boolean vc;
    Set<Integer> newVColSet = new HashSet<Integer>();
    for (int i = 0; i < hiveExprs.size(); i++) {
        pe = hiveExprs.get(i);
        rexN = calciteExprs.get(i);
        vc = false;
        if (rexN instanceof RexInputRef) {
            if (inpOpAf.vcolsInCalcite.contains(((RexInputRef) rexN).getIndex())) {
                newVColSet.add(i);
                vc = true;
            }
        }
        colInfos.add(new ColumnInfo(projNames.get(i), pe.getTypeInfo(), inpOpAf.tabAlias, vc));
    }
    return new Pair<ArrayList<ColumnInfo>, Set<Integer>>(colInfos, newVColSet);
}

Example 54 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class HiveProjectVisitor method genPTF.

private OpAttr genPTF(OpAttr inputOpAf, WindowingSpec wSpec) throws SemanticException {
    Operator<?> input = inputOpAf.inputs.get(0);
    wSpec.validateAndMakeEffective();
    WindowingComponentizer groups = new WindowingComponentizer(wSpec);
    RowResolver rr = new RowResolver();
    for (ColumnInfo ci : input.getSchema().getSignature()) {
        rr.put(inputOpAf.tabAlias, ci.getInternalName(), ci);
    }
    while (groups.hasNext()) {
        wSpec = groups.next(hiveOpConverter.getHiveConf(), hiveOpConverter.getSemanticAnalyzer(), hiveOpConverter.getUnparseTranslator(), rr);
        // 1. Create RS and backtrack Select operator on top
        ArrayList<ExprNodeDesc> keyCols = new ArrayList<ExprNodeDesc>();
        ArrayList<ExprNodeDesc> partCols = new ArrayList<ExprNodeDesc>();
        StringBuilder order = new StringBuilder();
        StringBuilder nullOrder = new StringBuilder();
        for (PartitionExpression partCol : wSpec.getQueryPartitionSpec().getExpressions()) {
            ExprNodeDesc partExpr = hiveOpConverter.getSemanticAnalyzer().genExprNodeDesc(partCol.getExpression(), rr);
            if (ExprNodeDescUtils.indexOf(partExpr, partCols) < 0) {
                keyCols.add(partExpr);
                partCols.add(partExpr);
                order.append('+');
                nullOrder.append('a');
            }
        }
        if (wSpec.getQueryOrderSpec() != null) {
            for (OrderExpression orderCol : wSpec.getQueryOrderSpec().getExpressions()) {
                ExprNodeDesc orderExpr = hiveOpConverter.getSemanticAnalyzer().genExprNodeDesc(orderCol.getExpression(), rr);
                char orderChar = orderCol.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-';
                char nullOrderChar = orderCol.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z';
                int index = ExprNodeDescUtils.indexOf(orderExpr, keyCols);
                if (index >= 0) {
                    order.setCharAt(index, orderChar);
                    nullOrder.setCharAt(index, nullOrderChar);
                    continue;
                }
                keyCols.add(orderExpr);
                order.append(orderChar);
                nullOrder.append(nullOrderChar);
            }
        }
        SelectOperator selectOp = genReduceSinkAndBacktrackSelect(input, keyCols.toArray(new ExprNodeDesc[keyCols.size()]), 0, partCols, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID, hiveOpConverter.getHiveConf());
        // 2. Finally create PTF
        PTFTranslator translator = new PTFTranslator();
        PTFDesc ptfDesc = translator.translate(wSpec, hiveOpConverter.getSemanticAnalyzer(), hiveOpConverter.getHiveConf(), rr, hiveOpConverter.getUnparseTranslator());
        RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
        Operator<?> ptfOp = OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), selectOp);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Generated " + ptfOp + " with row schema: [" + ptfOp.getSchema() + "]");
        }
        // 3. Prepare for next iteration (if any)
        rr = ptfOpRR;
        input = ptfOp;
    }
    return inputOpAf.clone(input);
}

Example 55 with ColumnInfo

use of org.apache.hadoop.hive.ql.exec.ColumnInfo in project hive by apache.

the class HiveUnionVisitor method genInputSelectForUnion.

private Operator<? extends OperatorDesc> genInputSelectForUnion(Operator<? extends OperatorDesc> origInputOp, ArrayList<ColumnInfo> uColumnInfo) throws SemanticException {
    Iterator<ColumnInfo> oIter = origInputOp.getSchema().getSignature().iterator();
    Iterator<ColumnInfo> uIter = uColumnInfo.iterator();
    List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>();
    List<String> colName = new ArrayList<String>();
    Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>();
    boolean needSelectOp = false;
    while (oIter.hasNext()) {
        ColumnInfo oInfo = oIter.next();
        ColumnInfo uInfo = uIter.next();
        if (!oInfo.isSameColumnForRR(uInfo)) {
            needSelectOp = true;
        }
        ExprNodeDesc column = new ExprNodeColumnDesc(oInfo.getType(), oInfo.getInternalName(), oInfo.getTabAlias(), oInfo.getIsVirtualCol(), oInfo.isSkewedCol());
        if (!oInfo.getType().equals(uInfo.getType())) {
            column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor().createConversionCast(column, (PrimitiveTypeInfo) uInfo.getType());
        }
        columns.add(column);
        colName.add(uInfo.getInternalName());
        columnExprMap.put(uInfo.getInternalName(), column);
    }
    if (needSelectOp) {
        return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(uColumnInfo), columnExprMap, origInputOp);
    } else {
        return origInputOp;
    }
}