Search in sources :

Example 16 with GenericUDFOPAnd

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd in project hive by apache.

the class TestVectorizationContext method testVectorizeFilterMultiAndOrExpression.

@Test
public void testVectorizeFilterMultiAndOrExpression() throws HiveException {
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(10));
    GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
    ExprNodeGenericFuncDesc greaterExprDesc = new ExprNodeGenericFuncDesc();
    greaterExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    greaterExprDesc.setGenericUDF(udf);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(constDesc);
    greaterExprDesc.setChildren(children1);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Float.class, "col2", "table", false);
    ExprNodeConstantDesc const2Desc = new ExprNodeConstantDesc(new Float(1.0));
    GenericUDFOPLessThan udf2 = new GenericUDFOPLessThan();
    ExprNodeGenericFuncDesc lessExprDesc = new ExprNodeGenericFuncDesc();
    lessExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    lessExprDesc.setGenericUDF(udf2);
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>(2);
    children2.add(col2Expr);
    children2.add(const2Desc);
    lessExprDesc.setChildren(children2);
    ExprNodeColumnDesc col3Expr = new ExprNodeColumnDesc(Integer.class, "col3", "table", false);
    ExprNodeConstantDesc const3Desc = new ExprNodeConstantDesc(new Integer(10));
    GenericUDFOPGreaterThan udf3 = new GenericUDFOPGreaterThan();
    ExprNodeGenericFuncDesc greaterExprDesc3 = new ExprNodeGenericFuncDesc();
    greaterExprDesc3.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    greaterExprDesc3.setGenericUDF(udf3);
    List<ExprNodeDesc> children3 = new ArrayList<ExprNodeDesc>(2);
    children3.add(col3Expr);
    children3.add(const3Desc);
    greaterExprDesc3.setChildren(children3);
    GenericUDFOPAnd andUdf = new GenericUDFOPAnd();
    ExprNodeGenericFuncDesc andExprDesc = new ExprNodeGenericFuncDesc();
    andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    andExprDesc.setGenericUDF(andUdf);
    List<ExprNodeDesc> children4 = new ArrayList<ExprNodeDesc>(2);
    children4.add(greaterExprDesc);
    children4.add(lessExprDesc);
    children4.add(greaterExprDesc3);
    andExprDesc.setChildren(children4);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    columns.add("col3");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(ve.getClass(), FilterExprAndExpr.class);
    assertEquals(ve.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
    assertEquals(ve.getChildExpressions()[1].getClass(), FilterDoubleColLessDoubleScalar.class);
    assertEquals(ve.getChildExpressions()[2].getClass(), FilterLongColGreaterLongScalar.class);
    GenericUDFOPOr orUdf = new GenericUDFOPOr();
    ExprNodeGenericFuncDesc orExprDesc = new ExprNodeGenericFuncDesc();
    orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    orExprDesc.setGenericUDF(orUdf);
    List<ExprNodeDesc> children5 = new ArrayList<ExprNodeDesc>(2);
    children5.add(greaterExprDesc);
    children5.add(lessExprDesc);
    children5.add(greaterExprDesc3);
    orExprDesc.setChildren(children5);
    VectorExpression veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(veOr.getClass(), FilterExprOrExpr.class);
    assertEquals(veOr.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
    assertEquals(veOr.getChildExpressions()[1].getClass(), FilterDoubleColLessDoubleScalar.class);
    assertEquals(ve.getChildExpressions()[2].getClass(), FilterLongColGreaterLongScalar.class);
}
Also used : GenericUDFOPGreaterThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) ArrayList(java.util.ArrayList) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) GenericUDFOPLessThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) VectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression) DynamicValueVectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.DynamicValueVectorExpression) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) GenericUDFOPOr(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr) GenericUDFOPAnd(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd) Test(org.junit.Test)

Example 17 with GenericUDFOPAnd

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd in project hive by apache.

the class TestVectorizationContext method testVectorizeFilterAndOrExpression.

@Test
public void testVectorizeFilterAndOrExpression() throws HiveException {
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(10));
    GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
    ExprNodeGenericFuncDesc greaterExprDesc = new ExprNodeGenericFuncDesc();
    greaterExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    greaterExprDesc.setGenericUDF(udf);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(constDesc);
    greaterExprDesc.setChildren(children1);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Float.class, "col2", "table", false);
    ExprNodeConstantDesc const2Desc = new ExprNodeConstantDesc(new Float(1.0));
    GenericUDFOPLessThan udf2 = new GenericUDFOPLessThan();
    ExprNodeGenericFuncDesc lessExprDesc = new ExprNodeGenericFuncDesc();
    lessExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    lessExprDesc.setGenericUDF(udf2);
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>(2);
    children2.add(col2Expr);
    children2.add(const2Desc);
    lessExprDesc.setChildren(children2);
    GenericUDFOPAnd andUdf = new GenericUDFOPAnd();
    ExprNodeGenericFuncDesc andExprDesc = new ExprNodeGenericFuncDesc();
    andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    andExprDesc.setGenericUDF(andUdf);
    List<ExprNodeDesc> children3 = new ArrayList<ExprNodeDesc>(2);
    children3.add(greaterExprDesc);
    children3.add(lessExprDesc);
    andExprDesc.setChildren(children3);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(ve.getClass(), FilterExprAndExpr.class);
    assertEquals(ve.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
    assertEquals(ve.getChildExpressions()[1].getClass(), FilterDoubleColLessDoubleScalar.class);
    GenericUDFOPOr orUdf = new GenericUDFOPOr();
    ExprNodeGenericFuncDesc orExprDesc = new ExprNodeGenericFuncDesc();
    orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    orExprDesc.setGenericUDF(orUdf);
    List<ExprNodeDesc> children4 = new ArrayList<ExprNodeDesc>(2);
    children4.add(greaterExprDesc);
    children4.add(lessExprDesc);
    orExprDesc.setChildren(children4);
    VectorExpression veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(veOr.getClass(), FilterExprOrExpr.class);
    assertEquals(veOr.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
    assertEquals(veOr.getChildExpressions()[1].getClass(), FilterDoubleColLessDoubleScalar.class);
}
Also used : GenericUDFOPGreaterThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) ArrayList(java.util.ArrayList) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) GenericUDFOPLessThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) VectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression) DynamicValueVectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.DynamicValueVectorExpression) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) GenericUDFOPOr(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr) GenericUDFOPAnd(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd) Test(org.junit.Test)

Example 18 with GenericUDFOPAnd

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd in project hive by apache.

the class ConstantPropagateProcFactory method shortcutFunction.

private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs, Operator<? extends Serializable> op) throws UDFArgumentException {
    if (udf instanceof GenericUDFOPEqual) {
        assert newExprs.size() == 2;
        boolean foundUDFInFirst = false;
        ExprNodeGenericFuncDesc caseOrWhenexpr = null;
        if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) {
            caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0);
            if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) {
                foundUDFInFirst = true;
            }
        }
        if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) {
            caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1);
            if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) {
                return null;
            }
        }
        if (null == caseOrWhenexpr) {
            // we didn't find case or when udf
            return null;
        }
        GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
        List<ExprNodeDesc> children = caseOrWhenexpr.getChildren();
        int i;
        if (childUDF instanceof GenericUDFWhen) {
            for (i = 1; i < children.size(); i += 2) {
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            if (children.size() % 2 == 1) {
                i = children.size() - 1;
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            // after constant folding of child expression the return type of UDFWhen might have changed,
            // so recreate the expression
            ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
            return newCaseOrWhenExpr;
        } else if (childUDF instanceof GenericUDFCase) {
            for (i = 2; i < children.size(); i += 2) {
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            if (children.size() % 2 == 0) {
                i = children.size() - 1;
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            // after constant folding of child expression the return type of UDFCase might have changed,
            // so recreate the expression
            ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
            return newCaseOrWhenExpr;
        } else {
            // cant happen
            return null;
        }
    }
    if (udf instanceof GenericUDFOPAnd) {
        final BitSet positionsToRemove = new BitSet();
        final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
        final List<Integer> notNullExprsPositions = new ArrayList<Integer>();
        final List<ExprNodeDesc> compareExprs = new ArrayList<ExprNodeDesc>();
        for (int i = 0; i < newExprs.size(); i++) {
            ExprNodeDesc childExpr = newExprs.get(i);
            if (childExpr instanceof ExprNodeConstantDesc) {
                ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
                if (Boolean.TRUE.equals(c.getValue())) {
                    // if true, prune it
                    positionsToRemove.set(i);
                } else {
                    // if false, return false
                    return childExpr;
                }
            } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
                notNullExprs.add(childExpr.getChildren().get(0));
                notNullExprsPositions.add(i);
            } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFBaseCompare && !(((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) && childExpr.getChildren().size() == 2) {
                // Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
                // where <op> can be "=", ">=", "<=", ">", "<".
                // Note: (key <> 86) and (key is not null) cannot be folded
                ExprNodeColumnDesc colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(0));
                if (null == colDesc) {
                    colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(1));
                }
                if (colDesc != null) {
                    compareExprs.add(colDesc);
                }
            }
        }
        // Try to fold (key = 86) and (key is not null) to (key = 86)
        for (int i = 0; i < notNullExprs.size(); i++) {
            for (ExprNodeDesc other : compareExprs) {
                if (notNullExprs.get(i).isSame(other)) {
                    positionsToRemove.set(notNullExprsPositions.get(i));
                    break;
                }
            }
        }
        // Remove unnecessary expressions
        int pos = 0;
        int removed = 0;
        while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
            newExprs.remove(pos - removed);
            pos++;
            removed++;
        }
        if (newExprs.size() == 0) {
            return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
        }
        if (newExprs.size() == 1) {
            return newExprs.get(0);
        }
    }
    if (udf instanceof GenericUDFOPOr) {
        final BitSet positionsToRemove = new BitSet();
        for (int i = 0; i < newExprs.size(); i++) {
            ExprNodeDesc childExpr = newExprs.get(i);
            if (childExpr instanceof ExprNodeConstantDesc) {
                ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
                if (Boolean.FALSE.equals(c.getValue())) {
                    // if false, prune it
                    positionsToRemove.set(i);
                } else if (Boolean.TRUE.equals(c.getValue())) {
                    // if true return true
                    return childExpr;
                }
            }
        }
        int pos = 0;
        int removed = 0;
        while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
            newExprs.remove(pos - removed);
            pos++;
            removed++;
        }
        if (newExprs.size() == 0) {
            return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.FALSE);
        }
        if (newExprs.size() == 1) {
            return newExprs.get(0);
        }
    }
    if (udf instanceof GenericUDFWhen) {
        if (!(newExprs.size() == 2 || newExprs.size() == 3)) {
            // we currently only handle either 1 or 2 branch.
            return null;
        }
        ExprNodeDesc thenExpr = newExprs.get(1);
        ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) : new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(), null);
        ExprNodeDesc whenExpr = newExprs.get(0);
        if (whenExpr instanceof ExprNodeConstantDesc) {
            Boolean whenVal = (Boolean) ((ExprNodeConstantDesc) whenExpr).getValue();
            return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr;
        }
        if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
            ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
            Object thenVal = constThen.getValue();
            Object elseVal = constElse.getValue();
            if (thenVal == null) {
                if (elseVal == null) {
                    // both branches are null.
                    return thenExpr;
                } else if (op instanceof FilterOperator) {
                    // we can still fold, since here null is equivalent to false.
                    return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
                } else {
                    // can't do much, expression is not in context of filter, so we can't treat null as equivalent to false here.
                    return null;
                }
            } else if (elseVal == null && op instanceof FilterOperator) {
                return Boolean.TRUE.equals(thenVal) ? whenExpr : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
            } else if (thenVal.equals(elseVal)) {
                return thenExpr;
            } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
                List<ExprNodeDesc> children = new ArrayList<>();
                children.add(whenExpr);
                children.add(new ExprNodeConstantDesc(false));
                ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children);
                if (Boolean.TRUE.equals(thenVal)) {
                    return func;
                } else {
                    List<ExprNodeDesc> exprs = new ArrayList<>();
                    exprs.add(func);
                    return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
                }
            } else {
                return null;
            }
        }
    }
    if (udf instanceof GenericUDFCase) {
        // where ss_sold_date= '1998-01-01' ;
        if (!(newExprs.size() == 3 || newExprs.size() == 4)) {
            // we currently only handle either 1 or 2 branch.
            return null;
        }
        ExprNodeDesc thenExpr = newExprs.get(2);
        ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) : new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(), null);
        if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
            ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
            Object thenVal = constThen.getValue();
            Object elseVal = constElse.getValue();
            if (thenVal == null) {
                if (null == elseVal) {
                    return thenExpr;
                } else if (op instanceof FilterOperator) {
                    return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
                } else {
                    return null;
                }
            } else if (null == elseVal && op instanceof FilterOperator) {
                return Boolean.TRUE.equals(thenVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
            } else if (thenVal.equals(elseVal)) {
                return thenExpr;
            } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
                ExprNodeGenericFuncDesc equal = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2));
                List<ExprNodeDesc> children = new ArrayList<>();
                children.add(equal);
                children.add(new ExprNodeConstantDesc(false));
                ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children);
                if (Boolean.TRUE.equals(thenVal)) {
                    return func;
                } else {
                    List<ExprNodeDesc> exprs = new ArrayList<>();
                    exprs.add(func);
                    return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
                }
            } else {
                return null;
            }
        }
    }
    if (udf instanceof GenericUDFUnixTimeStamp) {
        if (newExprs.size() >= 1) {
            // unix_timestamp(args) -> to_unix_timestamp(args)
            return ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), newExprs);
        }
    }
    return null;
}
Also used : GenericUDFCase(org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase) GenericUDFWhen(org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen) ArrayList(java.util.ArrayList) GenericUDFUnixTimeStamp(org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp) GenericUDFOPNotNull(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull) GenericUDFOPNotEqual(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual) GenericUDFBaseCompare(org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare) GenericUDFOPEqual(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) List(java.util.List) ArrayList(java.util.ArrayList) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) BitSet(java.util.BitSet) GenericUDFToUnixTimeStamp(org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) GenericUDFNvl(org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl) FilterOperator(org.apache.hadoop.hive.ql.exec.FilterOperator) GenericUDF(org.apache.hadoop.hive.ql.udf.generic.GenericUDF) DeferredJavaObject(org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject) GenericUDFOPNot(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot) GenericUDFOPOr(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr) GenericUDFOPAnd(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd)

Example 19 with GenericUDFOPAnd

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd in project hive by apache.

the class TestAccumuloPredicateHandler method testMultipleRanges.

@Test
public void testMultipleRanges() throws SerDeException {
    ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "rid", null, false);
    ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "aaa");
    List<ExprNodeDesc> children = Lists.newArrayList();
    children.add(column);
    children.add(constant);
    ExprNodeDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPEqualOrGreaterThan(), children);
    assertNotNull(node);
    ExprNodeDesc column2 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "rid", null, false);
    ExprNodeDesc constant2 = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "bbb");
    List<ExprNodeDesc> children2 = Lists.newArrayList();
    children2.add(column2);
    children2.add(constant2);
    ExprNodeDesc node2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPLessThan(), children2);
    assertNotNull(node2);
    List<ExprNodeDesc> bothFilters = Lists.newArrayList();
    bothFilters.add(node);
    bothFilters.add(node2);
    ExprNodeGenericFuncDesc both = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPAnd(), bothFilters);
    String filterExpr = SerializationUtilities.serializeExpression(both);
    conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, filterExpr);
    List<Range> ranges = handler.getRanges(conf, columnMapper);
    assertEquals(1, ranges.size());
    Range range = ranges.get(0);
    assertEquals(new Range(new Key("aaa"), true, new Key("bbb"), false), range);
}
Also used : ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Range(org.apache.accumulo.core.data.Range) GenericUDFOPLessThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan) GenericUDFOPEqualOrGreaterThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan) Key(org.apache.accumulo.core.data.Key) GenericUDFOPAnd(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd) Test(org.junit.Test)

Example 20 with GenericUDFOPAnd

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd in project hive by apache.

the class TestAccumuloPredicateHandler method testDisjointRanges.

@Test
public void testDisjointRanges() throws SerDeException {
    ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "rid", null, false);
    ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "aaa");
    List<ExprNodeDesc> children = Lists.newArrayList();
    children.add(column);
    children.add(constant);
    ExprNodeDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPEqualOrLessThan(), children);
    assertNotNull(node);
    ExprNodeDesc column2 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "rid", null, false);
    ExprNodeDesc constant2 = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "bbb");
    List<ExprNodeDesc> children2 = Lists.newArrayList();
    children2.add(column2);
    children2.add(constant2);
    ExprNodeDesc node2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPGreaterThan(), children2);
    assertNotNull(node2);
    List<ExprNodeDesc> bothFilters = Lists.newArrayList();
    bothFilters.add(node);
    bothFilters.add(node2);
    ExprNodeGenericFuncDesc both = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPAnd(), bothFilters);
    String filterExpr = SerializationUtilities.serializeExpression(both);
    conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, filterExpr);
    Collection<Range> ranges = handler.getRanges(conf, columnMapper);
    // Impossible to get ranges for row <= 'aaa' and row >= 'bbb'
    assertEquals(0, ranges.size());
}
Also used : GenericUDFOPGreaterThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) GenericUDFOPEqualOrLessThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Range(org.apache.accumulo.core.data.Range) GenericUDFOPAnd(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd) Test(org.junit.Test)

Aggregations

ExprNodeDesc (org.apache.hadoop.hive.ql.plan.ExprNodeDesc)23 ExprNodeGenericFuncDesc (org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc)23 GenericUDFOPAnd (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd)23 ExprNodeConstantDesc (org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc)21 ExprNodeColumnDesc (org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc)20 Test (org.junit.Test)19 ArrayList (java.util.ArrayList)13 GenericUDFOPEqualOrLessThan (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan)12 GenericUDFOPEqualOrGreaterThan (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan)10 List (java.util.List)9 Range (org.apache.accumulo.core.data.Range)9 GenericUDFOPGreaterThan (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan)9 GenericUDFOPOr (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr)8 HashMap (java.util.HashMap)7 DefaultGraphWalker (org.apache.hadoop.hive.ql.lib.DefaultGraphWalker)7 DefaultRuleDispatcher (org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher)7 Dispatcher (org.apache.hadoop.hive.ql.lib.Dispatcher)7 GraphWalker (org.apache.hadoop.hive.ql.lib.GraphWalker)7 Node (org.apache.hadoop.hive.ql.lib.Node)7 SemanticException (org.apache.hadoop.hive.ql.parse.SemanticException)7