Search in sources :

Example 16 with ExprNodeFieldDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc in project hive by apache.

the class IndexPredicateAnalyzer method analyzeExpr.

private ExprNodeDesc analyzeExpr(ExprNodeGenericFuncDesc expr, List<IndexSearchCondition> searchConditions, Object... nodeOutputs) throws SemanticException {
    if (FunctionRegistry.isOpAnd(expr)) {
        assert (nodeOutputs.length >= 2);
        List<ExprNodeDesc> residuals = new ArrayList<ExprNodeDesc>();
        for (Object residual : nodeOutputs) {
            if (null != residual) {
                residuals.add((ExprNodeDesc) residual);
            }
        }
        if (residuals.size() == 0) {
            return null;
        } else if (residuals.size() == 1) {
            return residuals.get(0);
        } else if (residuals.size() > 1) {
            return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, FunctionRegistry.getGenericUDFForAnd(), residuals);
        }
    }
    GenericUDF genericUDF = expr.getGenericUDF();
    if (!(genericUDF instanceof GenericUDFBaseCompare)) {
        return expr;
    }
    ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
    ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
    // We may need to peel off the GenericUDFBridge that is added by CBO or user
    if (expr1.getTypeInfo().equals(expr2.getTypeInfo())) {
        expr1 = getColumnExpr(expr1);
        expr2 = getColumnExpr(expr2);
    }
    ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
    if (extracted == null || (extracted.length > 2 && !acceptsFields)) {
        return expr;
    }
    ExprNodeColumnDesc columnDesc;
    ExprNodeConstantDesc constantDesc;
    if (extracted[0] instanceof ExprNodeConstantDesc) {
        genericUDF = genericUDF.flip();
        columnDesc = (ExprNodeColumnDesc) extracted[1];
        constantDesc = (ExprNodeConstantDesc) extracted[0];
    } else {
        columnDesc = (ExprNodeColumnDesc) extracted[0];
        constantDesc = (ExprNodeConstantDesc) extracted[1];
    }
    Set<String> allowed = columnToUDFs.get(columnDesc.getColumn());
    if (allowed == null) {
        return expr;
    }
    String udfName = genericUDF.getUdfName();
    if (!allowed.contains(genericUDF.getUdfName())) {
        return expr;
    }
    String[] fields = null;
    if (extracted.length > 2) {
        ExprNodeFieldDesc fieldDesc = (ExprNodeFieldDesc) extracted[2];
        if (!isValidField(fieldDesc)) {
            return expr;
        }
        fields = ExprNodeDescUtils.extractFields(fieldDesc);
    }
    // We also need to update the expr so that the index query can be generated.
    // Note that, hive does not support UDFToDouble etc in the query text.
    List<ExprNodeDesc> list = new ArrayList<ExprNodeDesc>();
    list.add(expr1);
    list.add(expr2);
    ExprNodeGenericFuncDesc indexExpr = new ExprNodeGenericFuncDesc(expr.getTypeInfo(), expr.getGenericUDF(), list);
    searchConditions.add(new IndexSearchCondition(columnDesc, udfName, constantDesc, indexExpr, expr, fields));
    // remove it from the residual predicate
    return fields == null ? null : expr;
}
Also used : ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) ArrayList(java.util.ArrayList) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) GenericUDFToString(org.apache.hadoop.hive.ql.udf.generic.GenericUDFToString) GenericUDF(org.apache.hadoop.hive.ql.udf.generic.GenericUDF) GenericUDFBaseCompare(org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare) ExprNodeFieldDesc(org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc)

Example 17 with ExprNodeFieldDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc in project hive by apache.

the class ExprNodeConverter method visitFieldAccess.

/**
 * TODO: Handle 1) cast 2), Windowing Agg Call
 */
@Override
public /*
   * Handles expr like struct(key,value).key
   * Follows same rules as TypeCheckProcFactory::getXpathOrFuncExprNodeDesc()
   * which is equivalent version of parsing such an expression from AST
   */
ExprNodeDesc visitFieldAccess(RexFieldAccess fieldAccess) {
    ExprNodeDesc parent = fieldAccess.getReferenceExpr().accept(this);
    String child = fieldAccess.getField().getName();
    TypeInfo parentType = parent.getTypeInfo();
    // Allow accessing a field of list element structs directly from a list
    boolean isList = (parentType.getCategory() == ObjectInspector.Category.LIST);
    if (isList) {
        parentType = ((ListTypeInfo) parentType).getListElementTypeInfo();
    }
    TypeInfo t = ((StructTypeInfo) parentType).getStructFieldTypeInfo(child);
    return new ExprNodeFieldDesc(t, parent, child, isList);
}
Also used : ExprNodeFieldDesc(org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) DateString(org.apache.calcite.util.DateString) ByteString(org.apache.calcite.avatica.util.ByteString) TimestampString(org.apache.calcite.util.TimestampString) TimeString(org.apache.calcite.util.TimeString) NlsString(org.apache.calcite.util.NlsString) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo)

Example 18 with ExprNodeFieldDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc in project hive by apache.

the class TestColumnPrunerProcCtx method testGetSelectNestedColPathsFromChildren1.

// Test select root.col1.a from root:struct<col1:struct<a:boolean,b:double>,col2:double>
@Test
public void testGetSelectNestedColPathsFromChildren1() {
    ColumnPrunerProcCtx ctx = new ColumnPrunerProcCtx(null);
    ExprNodeDesc colDesc = new ExprNodeColumnDesc(col3Type, "root", "test", false);
    ExprNodeDesc col1 = new ExprNodeFieldDesc(col1Type, colDesc, "col1", false);
    ExprNodeDesc fieldDesc = new ExprNodeFieldDesc(TypeInfoFactory.booleanTypeInfo, col1, "a", false);
    final List<FieldNode> paths = Arrays.asList(new FieldNode("_col0"));
    SelectOperator selectOperator = buildSelectOperator(Arrays.asList(fieldDesc), paths);
    List<FieldNode> groups = ctx.getSelectColsFromChildren(selectOperator, paths);
    compareTestResults(groups, "root.col1.a");
}
Also used : SelectOperator(org.apache.hadoop.hive.ql.exec.SelectOperator) ExprNodeFieldDesc(org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Test(org.junit.Test)

Example 19 with ExprNodeFieldDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc in project hive by apache.

the class TestColumnPrunerProcCtx method testGetSelectNestedColPathsFromChildren3.

// Test select root.col2 from root:struct<col1:struct<a:boolean,b:double>,col2:double>
@Test
public void testGetSelectNestedColPathsFromChildren3() {
    ColumnPrunerProcCtx ctx = new ColumnPrunerProcCtx(null);
    ExprNodeDesc colDesc = new ExprNodeColumnDesc(col3Type, "root", "test", false);
    ExprNodeDesc fieldDesc = new ExprNodeFieldDesc(col1Type, colDesc, "col2", false);
    final List<FieldNode> paths = Arrays.asList(new FieldNode("_col0"));
    SelectOperator selectOperator = buildSelectOperator(Arrays.asList(fieldDesc), paths);
    List<FieldNode> groups = ctx.getSelectColsFromChildren(selectOperator, paths);
    compareTestResults(groups, "root.col2");
}
Also used : SelectOperator(org.apache.hadoop.hive.ql.exec.SelectOperator) ExprNodeFieldDesc(org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Test(org.junit.Test)

Example 20 with ExprNodeFieldDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc in project hive by apache.

the class TestColumnPrunerProcCtx method testGetSelectNestedColPathsFromChildren7.

// Test select pow(root.col1.b, root.col2) from table test(root
// struct<col1:struct<a:boolean,b:double>, col2:double>);
@Test
public void testGetSelectNestedColPathsFromChildren7() {
    ColumnPrunerProcCtx ctx = new ColumnPrunerProcCtx(null);
    ExprNodeDesc colDesc = new ExprNodeColumnDesc(col3Type, "root", "test", false);
    ExprNodeDesc col1 = new ExprNodeFieldDesc(col1Type, colDesc, "col1", false);
    ExprNodeDesc fieldDesc1 = new ExprNodeFieldDesc(TypeInfoFactory.doubleTypeInfo, col1, "b", false);
    colDesc = new ExprNodeColumnDesc(col3Type, "root", "test", false);
    ExprNodeDesc col2 = new ExprNodeFieldDesc(col2Type, colDesc, "col2", false);
    final List<FieldNode> paths = Arrays.asList(new FieldNode("_col0"));
    GenericUDF udf = mock(GenericUDFPower.class);
    List<ExprNodeDesc> list = new ArrayList<>();
    list.add(fieldDesc1);
    list.add(col2);
    ExprNodeDesc funcDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.doubleTypeInfo, udf, "pow", list);
    SelectOperator selectOperator = buildSelectOperator(Arrays.asList(funcDesc), paths);
    List<FieldNode> groups = ctx.getSelectColsFromChildren(selectOperator, paths);
    compareTestResults(groups, "root.col1.b", "root.col2");
}
Also used : GenericUDF(org.apache.hadoop.hive.ql.udf.generic.GenericUDF) SelectOperator(org.apache.hadoop.hive.ql.exec.SelectOperator) ExprNodeFieldDesc(org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ArrayList(java.util.ArrayList) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Test(org.junit.Test)

Aggregations

ExprNodeFieldDesc (org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc)24 ExprNodeDesc (org.apache.hadoop.hive.ql.plan.ExprNodeDesc)23 ExprNodeColumnDesc (org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc)22 ExprNodeGenericFuncDesc (org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc)15 ArrayList (java.util.ArrayList)14 ExprNodeConstantDesc (org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc)13 SelectOperator (org.apache.hadoop.hive.ql.exec.SelectOperator)6 StructTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo)6 TypeInfo (org.apache.hadoop.hive.serde2.typeinfo.TypeInfo)6 Test (org.junit.Test)6 GenericUDF (org.apache.hadoop.hive.ql.udf.generic.GenericUDF)5 DecimalTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo)5 PrimitiveTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)5 DataTypePhysicalVariation (org.apache.hadoop.hive.common.type.DataTypePhysicalVariation)4 HiveException (org.apache.hadoop.hive.ql.metadata.HiveException)4 ExprNodeDynamicValueDesc (org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc)4 BaseCharTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo)4 ConstantVectorExpression (org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression)3 DynamicValueVectorExpression (org.apache.hadoop.hive.ql.exec.vector.expressions.DynamicValueVectorExpression)3 FilterConstantBooleanVectorExpression (org.apache.hadoop.hive.ql.exec.vector.expressions.FilterConstantBooleanVectorExpression)3