Examples with ExprDynamicParamDesc org.apache.hadoop.hive.ql.plan.ExprDynamicParamDesc used on opensource projects

Example 1 with ExprDynamicParamDesc

use of org.apache.hadoop.hive.ql.plan.ExprDynamicParamDesc in project hive by apache.

the class StatsUtils method getColStatisticsFromExpression.

/**
 * Get column statistics expression nodes
 * @param conf
 *          - hive conf
 * @param parentStats
 *          - parent statistics
 * @param end
 *          - expression nodes
 * @return column statistics
 */
public static ColStatistics getColStatisticsFromExpression(HiveConf conf, Statistics parentStats, ExprNodeDesc end) {
    if (end == null) {
        return null;
    }
    String colName = null;
    String colType = null;
    double avgColSize = 0;
    long countDistincts = 0;
    long numNulls = 0;
    ObjectInspector oi = end.getWritableObjectInspector();
    long numRows = parentStats.getNumRows();
    if (end instanceof ExprNodeColumnDesc) {
        // column projection
        ExprNodeColumnDesc encd = (ExprNodeColumnDesc) end;
        colName = encd.getColumn();
        if (encd.getIsPartitionColOrVirtualCol()) {
            ColStatistics colStats = parentStats.getColumnStatisticsFromColName(colName);
            if (colStats != null) {
                /* If statistics for the column already exist use it. */
                return colStats.clone();
            }
            // virtual columns
            colType = encd.getTypeInfo().getTypeName();
            countDistincts = numRows;
        } else {
            // clone the column stats and return
            ColStatistics result = parentStats.getColumnStatisticsFromColName(colName);
            if (result != null) {
                return result.clone();
            }
            return null;
        }
    } else if (end instanceof ExprNodeConstantDesc) {
        return buildColStatForConstant(conf, numRows, (ExprNodeConstantDesc) end);
    } else if (end instanceof ExprNodeGenericFuncDesc) {
        ExprNodeGenericFuncDesc engfd = (ExprNodeGenericFuncDesc) end;
        colName = engfd.getName();
        colType = engfd.getTypeString();
        // If it is a widening cast, we do not change NDV, min, max
        if (isWideningCast(engfd) && engfd.getChildren().get(0) instanceof ExprNodeColumnDesc) {
            // cast on single column
            ColStatistics stats = parentStats.getColumnStatisticsFromColName(engfd.getCols().get(0));
            if (stats != null) {
                ColStatistics newStats;
                newStats = stats.clone();
                newStats.setColumnName(colName);
                colType = colType.toLowerCase();
                newStats.setColumnType(colType);
                newStats.setAvgColLen(getAvgColLenOf(conf, oi, colType));
                return newStats;
            }
        }
        if (conf.getBoolVar(ConfVars.HIVE_STATS_ESTIMATORS_ENABLE)) {
            Optional<StatEstimatorProvider> sep = engfd.getGenericUDF().adapt(StatEstimatorProvider.class);
            if (sep.isPresent()) {
                StatEstimator se = sep.get().getStatEstimator();
                List<ColStatistics> csList = new ArrayList<ColStatistics>();
                for (ExprNodeDesc child : engfd.getChildren()) {
                    ColStatistics cs = getColStatisticsFromExpression(conf, parentStats, child);
                    if (cs == null) {
                        break;
                    }
                    csList.add(cs);
                }
                if (csList.size() == engfd.getChildren().size()) {
                    Optional<ColStatistics> res = se.estimate(csList);
                    if (res.isPresent()) {
                        ColStatistics newStats = res.get();
                        colType = colType.toLowerCase();
                        newStats.setColumnType(colType);
                        newStats.setColumnName(colName);
                        return newStats;
                    }
                }
            }
        }
        // fallback to default
        countDistincts = getNDVFor(engfd, numRows, parentStats);
    } else if (end instanceof ExprNodeColumnListDesc) {
        // column list
        ExprNodeColumnListDesc encd = (ExprNodeColumnListDesc) end;
        colName = Joiner.on(",").join(encd.getCols());
        colType = serdeConstants.LIST_TYPE_NAME;
        countDistincts = numRows;
    } else if (end instanceof ExprNodeFieldDesc) {
        // field within complex type
        ExprNodeFieldDesc enfd = (ExprNodeFieldDesc) end;
        colName = enfd.getFieldName();
        colType = enfd.getTypeString();
        countDistincts = numRows;
    } else if (end instanceof ExprDynamicParamDesc) {
        // possible to create colstats object
        return null;
    } else {
        throw new IllegalArgumentException("not supported expr type " + end.getClass());
    }
    colType = colType.toLowerCase();
    avgColSize = getAvgColLenOf(conf, oi, colType);
    ColStatistics colStats = new ColStatistics(colName, colType);
    colStats.setAvgColLen(avgColSize);
    colStats.setCountDistint(countDistincts);
    colStats.setNumNulls(numNulls);
    return colStats;
}

Example 2 with ExprDynamicParamDesc

use of org.apache.hadoop.hive.ql.plan.ExprDynamicParamDesc in project hive by apache.

the class ExecuteStatementAnalyzer method replaceDynamicParamsWithConstant.

/**
 * Given an expression tree root at expr and type info of the expression this method traverse
 * the expression tree and replaces all dynamic expression with the constant expression.
 * This method also does type inference for the new constant expression.
 * Note about type inference
 * Since dynamic parameter lacks type we need to figure out appropriate type to create constant
 * out of string value. To do this, we choose the type of first child of the parent expression
 * which isn't dynamic parameter
 */
private ExprNodeDesc replaceDynamicParamsWithConstant(ExprNodeDesc expr, TypeInfo typeInfo, Map<Integer, ASTNode> paramMap) throws SemanticException {
    if (expr.getChildren() == null || expr.getChildren().isEmpty()) {
        if (expr instanceof ExprDynamicParamDesc) {
            return getConstant((ExprDynamicParamDesc) expr, typeInfo, paramMap);
        }
        return expr;
    }
    for (ExprNodeDesc child : expr.getChildren()) {
        // we need typeinfo
        if (child instanceof ExprDynamicParamDesc) {
            continue;
        } else if (child.getTypeInfo() != TypeInfoFactory.voidTypeInfo) {
            typeInfo = child.getTypeInfo();
            break;
        }
    }
    Preconditions.checkArgument(typeInfo != null, "TypeInfo is null");
    List<ExprNodeDesc> exprList = new ArrayList<>();
    for (ExprNodeDesc child : expr.getChildren()) {
        child = replaceDynamicParamsWithConstant(child, typeInfo, paramMap);
        exprList.add(child);
    }
    expr.getChildren().clear();
    expr.getChildren().addAll(exprList);
    return expr;
}