Examples with RowSchema org.apache.hadoop.hive.ql.exec.RowSchema used on opensource projects

Example 51 with RowSchema

use of org.apache.hadoop.hive.ql.exec.RowSchema in project hive by apache.

the class TypeConverter method getType.

public static RelDataType getType(RelOptCluster cluster, RowResolver rr, List<String> neededCols) throws CalciteSemanticException {
    RexBuilder rexBuilder = cluster.getRexBuilder();
    RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
    RowSchema rs = rr.getRowSchema();
    List<RelDataType> fieldTypes = new LinkedList<RelDataType>();
    List<String> fieldNames = new LinkedList<String>();
    for (ColumnInfo ci : rs.getSignature()) {
        if (neededCols == null || neededCols.contains(ci.getInternalName())) {
            fieldTypes.add(convert(ci.getType(), dtFactory));
            fieldNames.add(ci.getInternalName());
        }
    }
    return dtFactory.createStructType(fieldTypes, fieldNames);
}

Example 52 with RowSchema

use of org.apache.hadoop.hive.ql.exec.RowSchema in project hive by apache.

the class CorrelationUtilities method replaceReduceSinkWithSelectOperator.

// replace the cRS to SEL operator
protected static SelectOperator replaceReduceSinkWithSelectOperator(ReduceSinkOperator childRS, ParseContext context, AbstractCorrelationProcCtx procCtx) throws SemanticException {
    RowSchema inputRS = childRS.getSchema();
    SelectDesc select = new SelectDesc(childRS.getConf().getValueCols(), childRS.getConf().getOutputValueColumnNames());
    Operator<?> parent = getSingleParent(childRS);
    parent.getChildOperators().clear();
    SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild(select, new RowSchema(inputRS.getSignature()), parent);
    sel.setColumnExprMap(childRS.getColumnExprMap());
    sel.setChildOperators(childRS.getChildOperators());
    for (Operator<? extends Serializable> ch : childRS.getChildOperators()) {
        ch.replaceParent(childRS, sel);
    }
    childRS.setChildOperators(null);
    childRS.setParentOperators(null);
    procCtx.addRemovedOperator(childRS);
    return sel;
}

Example 53 with RowSchema

use of org.apache.hadoop.hive.ql.exec.RowSchema in project hive by apache.

the class ColumnPrunerProcFactory method pruneOperator.

private static void pruneOperator(NodeProcessorCtx ctx, Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException {
    // the pruning needs to preserve the order of columns in the input schema
    RowSchema inputSchema = op.getSchema();
    if (inputSchema != null) {
        ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>();
        RowSchema oldRS = op.getSchema();
        for (ColumnInfo i : oldRS.getSignature()) {
            if (lookupColumn(cols, i.getInternalName()) != null) {
                rs.add(i);
            }
        }
        op.getSchema().setSignature(rs);
    }
}

Example 54 with RowSchema

use of org.apache.hadoop.hive.ql.exec.RowSchema in project hive by apache.

the class ColumnPrunerProcFactory method pruneReduceSinkOperator.

private static void pruneReduceSinkOperator(boolean[] retainFlags, ReduceSinkOperator reduce, ColumnPrunerProcCtx cppCtx) throws SemanticException {
    ReduceSinkDesc reduceConf = reduce.getConf();
    Map<String, ExprNodeDesc> oldMap = reduce.getColumnExprMap();
    LOG.info("RS " + reduce.getIdentifier() + " oldColExprMap: " + oldMap);
    RowSchema oldRS = reduce.getSchema();
    ArrayList<ColumnInfo> old_signature = oldRS.getSignature();
    ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>(old_signature);
    List<String> valueColNames = reduceConf.getOutputValueColumnNames();
    ArrayList<String> newValueColNames = new ArrayList<String>();
    List<ExprNodeDesc> keyExprs = reduceConf.getKeyCols();
    List<ExprNodeDesc> valueExprs = reduceConf.getValueCols();
    ArrayList<ExprNodeDesc> newValueExprs = new ArrayList<ExprNodeDesc>();
    for (int i = 0; i < retainFlags.length; i++) {
        String outputCol = valueColNames.get(i);
        ExprNodeDesc outputColExpr = valueExprs.get(i);
        if (!retainFlags[i]) {
            ColumnInfo colInfo = oldRS.getColumnInfo(outputCol);
            if (colInfo == null) {
                outputCol = Utilities.ReduceField.VALUE.toString() + "." + outputCol;
                colInfo = oldRS.getColumnInfo(outputCol);
            }
            // do row resolve once more because the ColumnInfo in row resolver is already removed
            if (colInfo == null) {
                continue;
            }
            // i.e. this column is not appearing in keyExprs of the RS
            if (ExprNodeDescUtils.indexOf(outputColExpr, keyExprs) == -1) {
                oldMap.remove(outputCol);
                signature.remove(colInfo);
            }
        } else {
            newValueColNames.add(outputCol);
            newValueExprs.add(outputColExpr);
        }
    }
    oldRS.setSignature(signature);
    reduce.getSchema().setSignature(signature);
    reduceConf.setOutputValueColumnNames(newValueColNames);
    reduceConf.setValueCols(newValueExprs);
    TableDesc newValueTable = PlanUtils.getReduceValueTableDesc(PlanUtils.getFieldSchemasFromColumnList(reduceConf.getValueCols(), newValueColNames, 0, ""));
    reduceConf.setValueSerializeInfo(newValueTable);
    LOG.info("RS " + reduce.getIdentifier() + " newColExprMap: " + oldMap);
}

Example 55 with RowSchema

use of org.apache.hadoop.hive.ql.exec.RowSchema in project hive by apache.

the class MapJoinProcessor method convertJoinOpMapJoinOp.

public static MapJoinOperator convertJoinOpMapJoinOp(HiveConf hconf, JoinOperator op, boolean leftInputJoin, String[] baseSrc, List<String> mapAliases, int mapJoinPos, boolean noCheckOuterJoin, boolean adjustParentsChildren) throws SemanticException {
    MapJoinDesc mapJoinDescriptor = getMapJoinDesc(hconf, op, leftInputJoin, baseSrc, mapAliases, mapJoinPos, noCheckOuterJoin, adjustParentsChildren);
    // reduce sink row resolver used to generate map join op
    RowSchema outputRS = op.getSchema();
    MapJoinOperator mapJoinOp = (MapJoinOperator) OperatorFactory.getAndMakeChild(op.getCompilationOpContext(), mapJoinDescriptor, new RowSchema(outputRS.getSignature()), op.getParentOperators());
    mapJoinOp.getConf().setReversedExprs(op.getConf().getReversedExprs());
    Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
    mapJoinOp.setColumnExprMap(colExprMap);
    List<Operator<? extends OperatorDesc>> childOps = op.getChildOperators();
    for (Operator<? extends OperatorDesc> childOp : childOps) {
        childOp.replaceParent(op, mapJoinOp);
    }
    mapJoinOp.setPosToAliasMap(op.getPosToAliasMap());
    mapJoinOp.setChildOperators(childOps);
    op.setChildOperators(null);
    op.setParentOperators(null);
    return mapJoinOp;
}