Examples with ExprNodeConstantDesc org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc used on opensource projects

Example 26 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TableAccessAnalyzer method genColNameMap.

/*
   * This method takes in an input operator and a subset of its output
   * column names, and generates the input column names for the operator
   * corresponding to those outputs. If the mapping from the input column
   * name to the output column name is not simple, the method returns
   * false, else it returns true. The list of output column names is
   * modified by this method to be the list of corresponding input column
   * names.
   */
private static boolean genColNameMap(Operator<? extends OperatorDesc> op, List<String> currColNames) {
    List<ExprNodeDesc> colList = null;
    List<String> outputColNames = null;
    assert (op.columnNamesRowResolvedCanBeObtained());
    // column names
    if (op instanceof SelectOperator) {
        SelectDesc selectDesc = ((SelectOperator) op).getConf();
        if (!selectDesc.isSelStarNoCompute()) {
            colList = selectDesc.getColList();
            outputColNames = selectDesc.getOutputColumnNames();
            // Only columns and constants can be selected
            for (int pos = 0; pos < colList.size(); pos++) {
                ExprNodeDesc colExpr = colList.get(pos);
                String outputColName = outputColNames.get(pos);
                // If it is not a column we need for the keys, move on
                if (!currColNames.contains(outputColName)) {
                    continue;
                }
                if (colExpr instanceof ExprNodeConstantDesc) {
                    currColNames.remove(outputColName);
                    continue;
                } else if (colExpr instanceof ExprNodeColumnDesc) {
                    String inputColName = ((ExprNodeColumnDesc) colExpr).getColumn();
                    if (!outputColName.equals(inputColName)) {
                        currColNames.set(currColNames.indexOf(outputColName), inputColName);
                    }
                } else {
                    // the column map can not be generated
                    return false;
                }
            }
        }
    }
    return true;
}

Example 27 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class ReduceSinkOperator method initializeOp.

@Override
protected void initializeOp(Configuration hconf) throws HiveException {
    super.initializeOp(hconf);
    try {
        numRows = 0;
        cntr = 1;
        logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
        statsMap.put(getCounterName(Counter.RECORDS_OUT_INTERMEDIATE, hconf), recordCounter);
        List<ExprNodeDesc> keys = conf.getKeyCols();
        if (isLogDebugEnabled) {
            LOG.debug("keys size is " + keys.size());
            for (ExprNodeDesc k : keys) {
                LOG.debug("Key exprNodeDesc " + k.getExprString());
            }
        }
        keyEval = new ExprNodeEvaluator[keys.size()];
        int i = 0;
        for (ExprNodeDesc e : keys) {
            if (e instanceof ExprNodeConstantDesc && (BUCKET_NUMBER_COL_NAME).equals(((ExprNodeConstantDesc) e).getValue())) {
                buckColIdxInKeyForSdpo = i;
            }
            keyEval[i++] = ExprNodeEvaluatorFactory.get(e);
        }
        numDistributionKeys = conf.getNumDistributionKeys();
        distinctColIndices = conf.getDistinctColumnIndices();
        numDistinctExprs = distinctColIndices.size();
        valueEval = new ExprNodeEvaluator[conf.getValueCols().size()];
        i = 0;
        for (ExprNodeDesc e : conf.getValueCols()) {
            valueEval[i++] = ExprNodeEvaluatorFactory.get(e);
        }
        partitionEval = new ExprNodeEvaluator[conf.getPartitionCols().size()];
        i = 0;
        for (ExprNodeDesc e : conf.getPartitionCols()) {
            int index = ExprNodeDescUtils.indexOf(e, keys);
            partitionEval[i++] = index < 0 ? ExprNodeEvaluatorFactory.get(e) : keyEval[index];
        }
        if (conf.getBucketCols() != null && !conf.getBucketCols().isEmpty()) {
            bucketEval = new ExprNodeEvaluator[conf.getBucketCols().size()];
            i = 0;
            for (ExprNodeDesc e : conf.getBucketCols()) {
                int index = ExprNodeDescUtils.indexOf(e, keys);
                bucketEval[i++] = index < 0 ? ExprNodeEvaluatorFactory.get(e) : keyEval[index];
            }
            buckColIdxInKey = conf.getPartitionCols().size();
        }
        tag = conf.getTag();
        tagByte[0] = (byte) tag;
        skipTag = conf.getSkipTag();
        if (isLogInfoEnabled) {
            LOG.info("Using tag = " + tag);
        }
        TableDesc keyTableDesc = conf.getKeySerializeInfo();
        keySerializer = (Serializer) keyTableDesc.getDeserializerClass().newInstance();
        keySerializer.initialize(null, keyTableDesc.getProperties());
        keyIsText = keySerializer.getSerializedClass().equals(Text.class);
        TableDesc valueTableDesc = conf.getValueSerializeInfo();
        valueSerializer = (Serializer) valueTableDesc.getDeserializerClass().newInstance();
        valueSerializer.initialize(null, valueTableDesc.getProperties());
        int limit = conf.getTopN();
        float memUsage = conf.getTopNMemoryUsage();
        if (limit >= 0 && memUsage > 0) {
            reducerHash = conf.isPTFReduceSink() ? new PTFTopNHash() : new TopNHash();
            reducerHash.initialize(limit, memUsage, conf.isMapGroupBy(), this, conf, hconf);
        }
        useUniformHash = conf.getReducerTraits().contains(UNIFORM);
        firstRow = true;
    } catch (Exception e) {
        String msg = "Error initializing ReduceSinkOperator: " + e.getMessage();
        LOG.error(msg, e);
        throw new RuntimeException(e);
    }
}

Example 28 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class ConstantPropagateProcFactory method evaluateFunction.

/**
   * Evaluate UDF
   *
   * @param udf UDF object
   * @param exprs
   * @param oldExprs
   * @return null if expression cannot be evaluated (not all parameters are constants). Or evaluated
   *         ExprNodeConstantDesc if possible.
   * @throws HiveException
   */
private static ExprNodeDesc evaluateFunction(GenericUDF udf, List<ExprNodeDesc> exprs, List<ExprNodeDesc> oldExprs) {
    DeferredJavaObject[] arguments = new DeferredJavaObject[exprs.size()];
    ObjectInspector[] argois = new ObjectInspector[exprs.size()];
    for (int i = 0; i < exprs.size(); i++) {
        ExprNodeDesc desc = exprs.get(i);
        if (desc instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constant = (ExprNodeConstantDesc) exprs.get(i);
            if (!constant.getTypeInfo().equals(oldExprs.get(i).getTypeInfo())) {
                constant = typeCast(constant, oldExprs.get(i).getTypeInfo());
                if (constant == null) {
                    return null;
                }
            }
            if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
                // nested complex types cannot be folded cleanly
                return null;
            }
            Object value = constant.getValue();
            PrimitiveTypeInfo pti = (PrimitiveTypeInfo) constant.getTypeInfo();
            Object writableValue = null == value ? value : PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti).getPrimitiveWritableObject(value);
            arguments[i] = new DeferredJavaObject(writableValue);
            argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
        } else if (desc instanceof ExprNodeGenericFuncDesc) {
            ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc) desc);
            if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) {
                return null;
            }
            ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn;
            if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
                // nested complex types cannot be folded cleanly
                return null;
            }
            Object writableValue = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((PrimitiveTypeInfo) constant.getTypeInfo()).getPrimitiveWritableObject(constant.getValue());
            arguments[i] = new DeferredJavaObject(writableValue);
            argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
        } else {
            return null;
        }
    }
    try {
        ObjectInspector oi = udf.initialize(argois);
        Object o = udf.evaluate(arguments);
        if (LOG.isDebugEnabled()) {
            LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o);
        }
        if (o == null) {
            return new ExprNodeConstantDesc(TypeInfoUtils.getTypeInfoFromObjectInspector(oi), o);
        }
        Class<?> clz = o.getClass();
        if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            TypeInfo typeInfo = poi.getTypeInfo();
            o = poi.getPrimitiveJavaObject(o);
            if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.VARCHAR_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) {
                return new ExprNodeConstantDesc(typeInfo, o);
            }
        } else if (udf instanceof GenericUDFStruct && oi instanceof StandardConstantStructObjectInspector) {
            // do not fold named_struct, only struct()
            ConstantObjectInspector coi = (ConstantObjectInspector) oi;
            TypeInfo structType = TypeInfoUtils.getTypeInfoFromObjectInspector(coi);
            return new ExprNodeConstantDesc(structType, ObjectInspectorUtils.copyToStandardJavaObject(o, coi));
        } else if (!PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) {
            if (LOG.isErrorEnabled()) {
                LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable.");
            }
            return null;
        } else {
        // fall through
        }
        String constStr = null;
        if (arguments.length == 1 && FunctionRegistry.isOpCast(udf)) {
            // remember original string representation of constant.
            constStr = arguments[0].get().toString();
        }
        return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr);
    } catch (HiveException e) {
        LOG.error("Evaluation function " + udf.getClass() + " failed in Constant Propagation Optimizer.");
        throw new RuntimeException(e);
    }
}

Example 29 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class ConstantPropagateProcFactory method propagate.

/**
   * Propagate assignment expression, adding an entry into constant map constants.
   *
   * @param udf expression UDF, currently only 2 UDFs are supported: '=' and 'is null'.
   * @param newExprs child expressions (parameters).
   * @param cppCtx
   * @param op
   * @param constants
   */
private static void propagate(GenericUDF udf, List<ExprNodeDesc> newExprs, RowSchema rs, Map<ColumnInfo, ExprNodeDesc> constants) {
    if (udf instanceof GenericUDFOPEqual) {
        ExprNodeDesc lOperand = newExprs.get(0);
        ExprNodeDesc rOperand = newExprs.get(1);
        ExprNodeConstantDesc v;
        if (lOperand instanceof ExprNodeConstantDesc) {
            v = (ExprNodeConstantDesc) lOperand;
        } else if (rOperand instanceof ExprNodeConstantDesc) {
            v = (ExprNodeConstantDesc) rOperand;
        } else {
            // we need a constant on one side.
            return;
        }
        // If both sides are constants, there is nothing to propagate
        ExprNodeColumnDesc c = ExprNodeDescUtils.getColumnExpr(lOperand);
        if (null == c) {
            c = ExprNodeDescUtils.getColumnExpr(rOperand);
        }
        if (null == c) {
            // we need a column expression on other side.
            return;
        }
        ColumnInfo ci = resolveColumn(rs, c);
        if (ci != null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
            }
            if (!v.getTypeInfo().equals(ci.getType())) {
                v = typeCast(v, ci.getType(), true);
            }
            if (v != null) {
                constants.put(ci, v);
            }
        }
    } else if (udf instanceof GenericUDFOPNull) {
        ExprNodeDesc operand = newExprs.get(0);
        if (operand instanceof ExprNodeColumnDesc) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
            }
            ExprNodeColumnDesc c = (ExprNodeColumnDesc) operand;
            ColumnInfo ci = resolveColumn(rs, c);
            if (ci != null) {
                constants.put(ci, new ExprNodeConstantDesc(ci.getType(), null));
            }
        }
    }
}

Example 30 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class PartitionPruner method compactExpr.

/**
   * Taking a partition pruning expression, remove the null operands and non-partition columns.
   * The reason why there are null operands is ExprProcFactory classes, for example
   * PPRColumnExprProcessor.
   * @param expr original partition pruning expression.
   * @return partition pruning expression that only contains partition columns.
   */
@VisibleForTesting
static ExprNodeDesc compactExpr(ExprNodeDesc expr) {
    // If this is a constant boolean expression, return the value.
    if (expr == null) {
        return null;
    }
    if (expr instanceof ExprNodeConstantDesc) {
        if (((ExprNodeConstantDesc) expr).getValue() == null)
            return null;
        if (!isBooleanExpr(expr)) {
            throw new IllegalStateException("Unexpected non-boolean ExprNodeConstantDesc: " + expr.getExprString());
        }
        return expr;
    } else if (expr instanceof ExprNodeGenericFuncDesc) {
        GenericUDF udf = ((ExprNodeGenericFuncDesc) expr).getGenericUDF();
        boolean isAnd = udf instanceof GenericUDFOPAnd;
        boolean isOr = udf instanceof GenericUDFOPOr;
        List<ExprNodeDesc> children = expr.getChildren();
        if (isAnd) {
            // Non-partition expressions are converted to nulls.
            List<ExprNodeDesc> newChildren = new ArrayList<ExprNodeDesc>();
            boolean allTrue = true;
            for (ExprNodeDesc child : children) {
                ExprNodeDesc compactChild = compactExpr(child);
                if (compactChild != null) {
                    if (!isTrueExpr(compactChild)) {
                        newChildren.add(compactChild);
                        allTrue = false;
                    }
                    if (isFalseExpr(compactChild)) {
                        return new ExprNodeConstantDesc(Boolean.FALSE);
                    }
                } else {
                    allTrue = false;
                }
            }
            if (allTrue) {
                return new ExprNodeConstantDesc(Boolean.TRUE);
            }
            if (newChildren.size() == 0) {
                return null;
            }
            if (newChildren.size() == 1) {
                return newChildren.get(0);
            }
            // Nothing to compact, update expr with compacted children.
            ((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
        } else if (isOr) {
            // Non-partition expressions are converted to nulls.
            List<ExprNodeDesc> newChildren = new ArrayList<ExprNodeDesc>();
            boolean allFalse = true;
            boolean isNull = false;
            for (ExprNodeDesc child : children) {
                ExprNodeDesc compactChild = compactExpr(child);
                if (compactChild != null) {
                    if (isTrueExpr(compactChild)) {
                        return new ExprNodeConstantDesc(Boolean.TRUE);
                    }
                    if (!isNull && !isFalseExpr(compactChild)) {
                        newChildren.add(compactChild);
                        allFalse = false;
                    }
                } else {
                    isNull = true;
                }
            }
            if (isNull) {
                return null;
            }
            if (allFalse) {
                return new ExprNodeConstantDesc(Boolean.FALSE);
            }
            if (newChildren.size() == 1) {
                return newChildren.get(0);
            }
            // Nothing to compact, update expr with compacted children.
            ((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
        }
        return expr;
    } else {
        throw new IllegalStateException("Unexpected type of ExprNodeDesc: " + expr.getExprString());
    }
}