Examples with ExprNodeDesc org.apache.hadoop.hive.ql.plan.ExprNodeDesc used on opensource projects

Example 96 with ExprNodeDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeDesc in project hive by apache.

the class ConstantPropagateProcFactory method evaluateFunction.

/**
   * Evaluate UDF
   *
   * @param udf UDF object
   * @param exprs
   * @param oldExprs
   * @return null if expression cannot be evaluated (not all parameters are constants). Or evaluated
   *         ExprNodeConstantDesc if possible.
   * @throws HiveException
   */
private static ExprNodeDesc evaluateFunction(GenericUDF udf, List<ExprNodeDesc> exprs, List<ExprNodeDesc> oldExprs) {
    DeferredJavaObject[] arguments = new DeferredJavaObject[exprs.size()];
    ObjectInspector[] argois = new ObjectInspector[exprs.size()];
    for (int i = 0; i < exprs.size(); i++) {
        ExprNodeDesc desc = exprs.get(i);
        if (desc instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constant = (ExprNodeConstantDesc) exprs.get(i);
            if (!constant.getTypeInfo().equals(oldExprs.get(i).getTypeInfo())) {
                constant = typeCast(constant, oldExprs.get(i).getTypeInfo());
                if (constant == null) {
                    return null;
                }
            }
            if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
                // nested complex types cannot be folded cleanly
                return null;
            }
            Object value = constant.getValue();
            PrimitiveTypeInfo pti = (PrimitiveTypeInfo) constant.getTypeInfo();
            Object writableValue = null == value ? value : PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti).getPrimitiveWritableObject(value);
            arguments[i] = new DeferredJavaObject(writableValue);
            argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
        } else if (desc instanceof ExprNodeGenericFuncDesc) {
            ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc) desc);
            if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) {
                return null;
            }
            ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn;
            if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
                // nested complex types cannot be folded cleanly
                return null;
            }
            Object writableValue = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((PrimitiveTypeInfo) constant.getTypeInfo()).getPrimitiveWritableObject(constant.getValue());
            arguments[i] = new DeferredJavaObject(writableValue);
            argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
        } else {
            return null;
        }
    }
    try {
        ObjectInspector oi = udf.initialize(argois);
        Object o = udf.evaluate(arguments);
        if (LOG.isDebugEnabled()) {
            LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o);
        }
        if (o == null) {
            return new ExprNodeConstantDesc(TypeInfoUtils.getTypeInfoFromObjectInspector(oi), o);
        }
        Class<?> clz = o.getClass();
        if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            TypeInfo typeInfo = poi.getTypeInfo();
            o = poi.getPrimitiveJavaObject(o);
            if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.VARCHAR_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) {
                return new ExprNodeConstantDesc(typeInfo, o);
            }
        } else if (udf instanceof GenericUDFStruct && oi instanceof StandardConstantStructObjectInspector) {
            // do not fold named_struct, only struct()
            ConstantObjectInspector coi = (ConstantObjectInspector) oi;
            TypeInfo structType = TypeInfoUtils.getTypeInfoFromObjectInspector(coi);
            return new ExprNodeConstantDesc(structType, ObjectInspectorUtils.copyToStandardJavaObject(o, coi));
        } else if (!PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) {
            if (LOG.isErrorEnabled()) {
                LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable.");
            }
            return null;
        } else {
        // fall through
        }
        String constStr = null;
        if (arguments.length == 1 && FunctionRegistry.isOpCast(udf)) {
            // remember original string representation of constant.
            constStr = arguments[0].get().toString();
        }
        return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr);
    } catch (HiveException e) {
        LOG.error("Evaluation function " + udf.getClass() + " failed in Constant Propagation Optimizer.");
        throw new RuntimeException(e);
    }
}

Example 97 with ExprNodeDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeDesc in project hive by apache.

the class ConstantPropagateProcFactory method foldOperator.

/**
   * Change operator row schema, replace column with constant if it is.
   *
   * @param op
   * @param constants
   * @throws SemanticException
   */
private static void foldOperator(Operator<? extends Serializable> op, ConstantPropagateProcCtx cppCtx) throws SemanticException {
    RowSchema schema = op.getSchema();
    Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getOpToConstantExprs().get(op);
    if (schema != null && schema.getSignature() != null) {
        for (ColumnInfo col : schema.getSignature()) {
            ExprNodeDesc constant = constants.get(col);
            if (constant != null) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Replacing column " + col + " with constant " + constant + " in " + op);
                }
                if (!col.getType().equals(constant.getTypeInfo())) {
                    constant = typeCast(constant, col.getType());
                }
                if (constant != null) {
                    col.setObjectinspector(constant.getWritableObjectInspector());
                }
            }
        }
    }
    Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
    if (colExprMap != null) {
        for (Entry<ColumnInfo, ExprNodeDesc> e : constants.entrySet()) {
            String internalName = e.getKey().getInternalName();
            if (colExprMap.containsKey(internalName)) {
                colExprMap.put(internalName, e.getValue());
            }
        }
    }
}

Example 98 with ExprNodeDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeDesc in project hive by apache.

the class ConstantPropagateProcFactory method propagate.

/**
   * Propagate assignment expression, adding an entry into constant map constants.
   *
   * @param udf expression UDF, currently only 2 UDFs are supported: '=' and 'is null'.
   * @param newExprs child expressions (parameters).
   * @param cppCtx
   * @param op
   * @param constants
   */
private static void propagate(GenericUDF udf, List<ExprNodeDesc> newExprs, RowSchema rs, Map<ColumnInfo, ExprNodeDesc> constants) {
    if (udf instanceof GenericUDFOPEqual) {
        ExprNodeDesc lOperand = newExprs.get(0);
        ExprNodeDesc rOperand = newExprs.get(1);
        ExprNodeConstantDesc v;
        if (lOperand instanceof ExprNodeConstantDesc) {
            v = (ExprNodeConstantDesc) lOperand;
        } else if (rOperand instanceof ExprNodeConstantDesc) {
            v = (ExprNodeConstantDesc) rOperand;
        } else {
            // we need a constant on one side.
            return;
        }
        // If both sides are constants, there is nothing to propagate
        ExprNodeColumnDesc c = ExprNodeDescUtils.getColumnExpr(lOperand);
        if (null == c) {
            c = ExprNodeDescUtils.getColumnExpr(rOperand);
        }
        if (null == c) {
            // we need a column expression on other side.
            return;
        }
        ColumnInfo ci = resolveColumn(rs, c);
        if (ci != null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
            }
            if (!v.getTypeInfo().equals(ci.getType())) {
                v = typeCast(v, ci.getType(), true);
            }
            if (v != null) {
                constants.put(ci, v);
            }
        }
    } else if (udf instanceof GenericUDFOPNull) {
        ExprNodeDesc operand = newExprs.get(0);
        if (operand instanceof ExprNodeColumnDesc) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
            }
            ExprNodeColumnDesc c = (ExprNodeColumnDesc) operand;
            ColumnInfo ci = resolveColumn(rs, c);
            if (ci != null) {
                constants.put(ci, new ExprNodeConstantDesc(ci.getType(), null));
            }
        }
    }
}

Example 99 with ExprNodeDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeDesc in project hive by apache.

the class CalcitePlanner method handleInsertStatement.

// This function serves as the wrapper of handleInsertStatementSpec in
// SemanticAnalyzer
Operator<?> handleInsertStatement(String dest, Operator<?> input, RowResolver inputRR, QB qb) throws SemanticException {
    ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
    ArrayList<ColumnInfo> columns = inputRR.getColumnInfos();
    for (int i = 0; i < columns.size(); i++) {
        ColumnInfo col = columns.get(i);
        colList.add(new ExprNodeColumnDesc(col));
    }
    ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
    RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, inputRR, qb, selExprList);
    ArrayList<String> columnNames = new ArrayList<String>();
    Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
    for (int i = 0; i < colList.size(); i++) {
        String outputCol = getColumnInternalName(i);
        colExprMap.put(outputCol, colList.get(i));
        columnNames.add(outputCol);
    }
    Operator<?> output = putOpInsertMap(OperatorFactory.getAndMakeChild(new SelectDesc(colList, columnNames), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch);
    output.setColumnExprMap(colExprMap);
    return output;
}

Example 100 with ExprNodeDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeDesc in project hive by apache.

the class BucketingSortingOpProcFactory method findBucketingSortingColumns.

/**
   * For each expression, check if it represents a column known to be bucketed/sorted.
   *
   * The methods setBucketingColsIfComplete and setSortingColsIfComplete should be used to assign
   * the values of newBucketCols and newSortCols as the bucketing/sorting columns of this operator
   * because these arrays may contain nulls indicating that the output of this operator is not
   * bucketed/sorted.
   *
   * @param exprs - list of expression
   * @param colInfos - list of column infos
   * @param bucketCols - list of bucketed columns from the input
   * @param sortCols - list of sorted columns from the input
   * @param newBucketCols - an array of bucket columns which should be the same length as
   *    bucketCols, updated such that the bucketed column(s) at index i in bucketCols became
   *    the bucketed column(s) at index i of newBucketCols in the output
   * @param newSortCols - an array of sort columns which should be the same length as
   *    sortCols, updated such that the sorted column(s) at index i in sortCols became
   *    the sorted column(s) at index i of sortCols in the output
   * @param colInfosOffset - the expressions are known to be represented by column infos
   *    beginning at this index
   */
private static void findBucketingSortingColumns(List<ExprNodeDesc> exprs, List<ColumnInfo> colInfos, List<BucketCol> bucketCols, List<SortCol> sortCols, BucketCol[] newBucketCols, SortCol[] newSortCols, int colInfosOffset) {
    for (int cnt = 0; cnt < exprs.size(); cnt++) {
        ExprNodeDesc expr = exprs.get(cnt);
        // voids any assumptions
        if (!(expr instanceof ExprNodeColumnDesc)) {
            continue;
        }
        ExprNodeColumnDesc columnExpr = (ExprNodeColumnDesc) expr;
        int colInfosIndex = cnt + colInfosOffset;
        if (newBucketCols != null) {
            int bucketIndex = indexOfColName(bucketCols, columnExpr.getColumn());
            if (bucketIndex != -1) {
                if (newBucketCols[bucketIndex] == null) {
                    newBucketCols[bucketIndex] = new BucketCol();
                }
                newBucketCols[bucketIndex].addAlias(colInfos.get(colInfosIndex).getInternalName(), colInfosIndex);
            }
        }
        if (newSortCols != null) {
            int sortIndex = indexOfColName(sortCols, columnExpr.getColumn());
            if (sortIndex != -1) {
                if (newSortCols[sortIndex] == null) {
                    newSortCols[sortIndex] = new SortCol(sortCols.get(sortIndex).getSortOrder());
                }
                newSortCols[sortIndex].addAlias(colInfos.get(colInfosIndex).getInternalName(), colInfosIndex);
            }
        }
    }
}