Examples with ExprNodeColumnListDesc org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc used on opensource projects

Example 1 with ExprNodeColumnListDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc in project hive by apache.

the class ExprNodeDescExprFactory method addExprToExprsList.

/**
 * {@inheritDoc}
 */
@Override
protected void addExprToExprsList(ExprNodeDesc columnList, ExprNodeDesc expr) {
    ExprNodeColumnListDesc l = (ExprNodeColumnListDesc) columnList;
    l.addColumn(expr);
}

Example 2 with ExprNodeColumnListDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc in project hive by apache.

the class StatsUtils method getColStatisticsFromExpression.

/**
 * Get column statistics expression nodes
 * @param conf
 *          - hive conf
 * @param parentStats
 *          - parent statistics
 * @param end
 *          - expression nodes
 * @return column statistics
 */
public static ColStatistics getColStatisticsFromExpression(HiveConf conf, Statistics parentStats, ExprNodeDesc end) {
    if (end == null) {
        return null;
    }
    String colName = null;
    String colType = null;
    double avgColSize = 0;
    long countDistincts = 0;
    long numNulls = 0;
    ObjectInspector oi = end.getWritableObjectInspector();
    long numRows = parentStats.getNumRows();
    if (end instanceof ExprNodeColumnDesc) {
        // column projection
        ExprNodeColumnDesc encd = (ExprNodeColumnDesc) end;
        colName = encd.getColumn();
        if (encd.getIsPartitionColOrVirtualCol()) {
            ColStatistics colStats = parentStats.getColumnStatisticsFromColName(colName);
            if (colStats != null) {
                /* If statistics for the column already exist use it. */
                return colStats.clone();
            }
            // virtual columns
            colType = encd.getTypeInfo().getTypeName();
            countDistincts = numRows;
        } else {
            // clone the column stats and return
            ColStatistics result = parentStats.getColumnStatisticsFromColName(colName);
            if (result != null) {
                return result.clone();
            }
            return null;
        }
    } else if (end instanceof ExprNodeConstantDesc) {
        return buildColStatForConstant(conf, numRows, (ExprNodeConstantDesc) end);
    } else if (end instanceof ExprNodeGenericFuncDesc) {
        ExprNodeGenericFuncDesc engfd = (ExprNodeGenericFuncDesc) end;
        colName = engfd.getName();
        colType = engfd.getTypeString();
        // If it is a widening cast, we do not change NDV, min, max
        if (isWideningCast(engfd) && engfd.getChildren().get(0) instanceof ExprNodeColumnDesc) {
            // cast on single column
            ColStatistics stats = parentStats.getColumnStatisticsFromColName(engfd.getCols().get(0));
            if (stats != null) {
                ColStatistics newStats;
                newStats = stats.clone();
                newStats.setColumnName(colName);
                colType = colType.toLowerCase();
                newStats.setColumnType(colType);
                newStats.setAvgColLen(getAvgColLenOf(conf, oi, colType));
                return newStats;
            }
        }
        if (conf.getBoolVar(ConfVars.HIVE_STATS_ESTIMATORS_ENABLE)) {
            Optional<StatEstimatorProvider> sep = engfd.getGenericUDF().adapt(StatEstimatorProvider.class);
            if (sep.isPresent()) {
                StatEstimator se = sep.get().getStatEstimator();
                List<ColStatistics> csList = new ArrayList<ColStatistics>();
                for (ExprNodeDesc child : engfd.getChildren()) {
                    ColStatistics cs = getColStatisticsFromExpression(conf, parentStats, child);
                    if (cs == null) {
                        break;
                    }
                    csList.add(cs);
                }
                if (csList.size() == engfd.getChildren().size()) {
                    Optional<ColStatistics> res = se.estimate(csList);
                    if (res.isPresent()) {
                        ColStatistics newStats = res.get();
                        colType = colType.toLowerCase();
                        newStats.setColumnType(colType);
                        newStats.setColumnName(colName);
                        return newStats;
                    }
                }
            }
        }
        // fallback to default
        countDistincts = getNDVFor(engfd, numRows, parentStats);
    } else if (end instanceof ExprNodeColumnListDesc) {
        // column list
        ExprNodeColumnListDesc encd = (ExprNodeColumnListDesc) end;
        colName = Joiner.on(",").join(encd.getCols());
        colType = serdeConstants.LIST_TYPE_NAME;
        countDistincts = numRows;
    } else if (end instanceof ExprNodeFieldDesc) {
        // field within complex type
        ExprNodeFieldDesc enfd = (ExprNodeFieldDesc) end;
        colName = enfd.getFieldName();
        colType = enfd.getTypeString();
        countDistincts = numRows;
    } else if (end instanceof ExprDynamicParamDesc) {
        // possible to create colstats object
        return null;
    } else {
        throw new IllegalArgumentException("not supported expr type " + end.getClass());
    }
    colType = colType.toLowerCase();
    avgColSize = getAvgColLenOf(conf, oi, colType);
    ColStatistics colStats = new ColStatistics(colName, colType);
    colStats.setAvgColLen(avgColSize);
    colStats.setCountDistint(countDistincts);
    colStats.setNumNulls(numNulls);
    return colStats;
}

Example 3 with ExprNodeColumnListDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc in project hive by apache.

the class SemanticAnalyzer method genAllExprNodeDesc.

/**
 * Generates all of the expression node descriptors for the expression and children of it
 * passed in the arguments. This function uses the row resolver and the metadata information
 * that are passed as arguments to resolve the column names to internal names.
 *
 * @param expr
 *          The expression
 * @param input
 *          The row resolver
 * @param tcCtx
 *          Customized type-checking context
 * @return expression to exprNodeDesc mapping
 * @throws SemanticException Failed to evaluate expression
 */
@SuppressWarnings("nls")
Map<ASTNode, ExprNodeDesc> genAllExprNodeDesc(ASTNode expr, RowResolver input, TypeCheckCtx tcCtx) throws SemanticException {
    // Create the walker and  the rules dispatcher.
    tcCtx.setUnparseTranslator(unparseTranslator);
    Map<ASTNode, ExprNodeDesc> nodeOutputs = ExprNodeTypeCheck.genExprNode(expr, tcCtx);
    ExprNodeDesc desc = nodeOutputs.get(expr);
    if (desc == null) {
        String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
        ColumnInfo colInfo = input.get(null, tableOrCol);
        String errMsg;
        if (colInfo == null && input.getIsExprResolver()) {
            errMsg = ASTErrorUtils.getMsg(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(), expr);
        } else {
            errMsg = tcCtx.getError();
        }
        throw new SemanticException(Optional.ofNullable(errMsg).orElse("Error in parsing "));
    }
    if (desc instanceof ExprNodeColumnListDesc) {
        throw new SemanticException("TOK_ALLCOLREF is not supported in current context");
    }
    if (!unparseTranslator.isEnabled()) {
        // Not creating a view, so no need to track view expansions.
        return nodeOutputs;
    }
    List<ASTNode> fieldDescList = new ArrayList<>();
    for (Map.Entry<ASTNode, ExprNodeDesc> entry : nodeOutputs.entrySet()) {
        if (!(entry.getValue() instanceof ExprNodeColumnDesc)) {
            // struct<>.
            if (entry.getValue() instanceof ExprNodeFieldDesc) {
                fieldDescList.add(entry.getKey());
            }
            continue;
        }
        ASTNode node = entry.getKey();
        ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) entry.getValue();
        if ((columnDesc.getTabAlias() == null) || (columnDesc.getTabAlias().length() == 0)) {
            // internal expressions used in the representation of aggregation.
            continue;
        }
        String[] tmp = input.reverseLookup(columnDesc.getColumn());
        // check if outer present && (tmp is null || tmp not null - contains tbl info)
        if (tcCtx.getOuterRR() != null && (tmp == null || (tmp[0] != null && columnDesc.getTabAlias() != null && !tmp[0].equals(columnDesc.getTabAlias())))) {
            tmp = tcCtx.getOuterRR().reverseLookup(columnDesc.getColumn());
        }
        StringBuilder replacementText = new StringBuilder();
        replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf));
        replacementText.append(".");
        replacementText.append(HiveUtils.unparseIdentifier(tmp[1], conf));
        unparseTranslator.addTranslation(node, replacementText.toString());
    }
    for (ASTNode node : fieldDescList) {
        Map<ASTNode, String> map = translateFieldDesc(node);
        for (Entry<ASTNode, String> entry : map.entrySet()) {
            unparseTranslator.addTranslation(entry.getKey(), entry.getValue().toLowerCase());
        }
    }
    return nodeOutputs;
}