use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull in project hive by apache.
the class ConstantPropagateProcFactory method foldNegative.
/**
* Combines the logical not() operator with the child operator if possible.
* @param desc the expression to be evaluated
* @return the new expression to be replaced
* @throws UDFArgumentException
*/
private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException {
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
GenericUDF udf = funcDesc.getGenericUDF();
if (udf instanceof GenericUDFOPNot) {
ExprNodeDesc child = funcDesc.getChildren().get(0);
if (child instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc) child;
GenericUDF childUDF = childDesc.getGenericUDF();
List<ExprNodeDesc> grandChildren = child.getChildren();
if (childUDF instanceof GenericUDFBaseCompare || childUDF instanceof GenericUDFOPNull || childUDF instanceof GenericUDFOPNotNull) {
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(grandChild));
}
return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren);
} else if (childUDF instanceof GenericUDFOPAnd || childUDF instanceof GenericUDFOPOr) {
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), Arrays.asList(grandChild))));
}
return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren);
} else if (childUDF instanceof GenericUDFOPNot) {
return foldNegative(child.getChildren().get(0));
} else {
// For operator like if() that cannot be handled, leave not() as it
// is and continue processing the children
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(grandChild));
}
childDesc.setChildren(newGrandChildren);
return funcDesc;
}
}
}
}
return desc;
}
use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull in project phoenix by apache.
the class IndexPredicateAnalyzer method analyzeExpr.
private ExprNodeDesc analyzeExpr(ExprNodeGenericFuncDesc expr, List<IndexSearchCondition> searchConditions, Object... nodeOutputs) throws SemanticException {
if (FunctionRegistry.isOpAnd(expr)) {
assert (nodeOutputs.length == 2);
ExprNodeDesc residual1 = (ExprNodeDesc) nodeOutputs[0];
ExprNodeDesc residual2 = (ExprNodeDesc) nodeOutputs[1];
if (residual1 == null) {
return residual2;
}
if (residual2 == null) {
return residual1;
}
List<ExprNodeDesc> residuals = new ArrayList<ExprNodeDesc>();
residuals.add(residual1);
residuals.add(residual2);
return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, FunctionRegistry.getGenericUDFForAnd(), residuals);
}
GenericUDF genericUDF = expr.getGenericUDF();
if (!(genericUDF instanceof GenericUDFBaseCompare)) {
// 2015-10-22 Added by JeongMin Ju : Processing Between/In Operator
if (genericUDF instanceof GenericUDFBetween) {
// In case of not between, The value of first element of nodeOutputs is true.
// otherwise false.
processingBetweenOperator(expr, searchConditions, nodeOutputs);
return expr;
} else if (genericUDF instanceof GenericUDFIn) {
// In case of not in operator, in operator exist as child of not operator.
processingInOperator(expr, searchConditions, false, nodeOutputs);
return expr;
} else if (genericUDF instanceof GenericUDFOPNot && ((ExprNodeGenericFuncDesc) expr.getChildren().get(0)).getGenericUDF() instanceof GenericUDFIn) {
// In case of not in operator, in operator exist as child of not operator.
processingInOperator((ExprNodeGenericFuncDesc) expr.getChildren().get(0), searchConditions, true, ((ExprNodeGenericFuncDesc) nodeOutputs[0]).getChildren().toArray());
return expr;
} else if (genericUDF instanceof GenericUDFOPNull) {
processingNullOperator(expr, searchConditions, nodeOutputs);
return expr;
} else if (genericUDF instanceof GenericUDFOPNotNull) {
processingNotNullOperator(expr, searchConditions, nodeOutputs);
return expr;
} else {
return expr;
}
}
ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
// user
if (expr1.getTypeInfo().equals(expr2.getTypeInfo())) {
expr1 = getColumnExpr(expr1);
expr2 = getColumnExpr(expr2);
}
ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
if (extracted == null || (extracted.length > 2 && !acceptsFields)) {
return expr;
}
ExprNodeColumnDesc columnDesc;
ExprNodeConstantDesc constantDesc;
if (extracted[0] instanceof ExprNodeConstantDesc) {
genericUDF = genericUDF.flip();
columnDesc = (ExprNodeColumnDesc) extracted[1];
constantDesc = (ExprNodeConstantDesc) extracted[0];
} else {
columnDesc = (ExprNodeColumnDesc) extracted[0];
constantDesc = (ExprNodeConstantDesc) extracted[1];
}
Set<String> allowed = columnToUDFs.get(columnDesc.getColumn());
if (allowed == null) {
return expr;
}
String udfName = genericUDF.getUdfName();
if (!allowed.contains(genericUDF.getUdfName())) {
return expr;
}
String[] fields = null;
if (extracted.length > 2) {
ExprNodeFieldDesc fieldDesc = (ExprNodeFieldDesc) extracted[2];
if (!isValidField(fieldDesc)) {
return expr;
}
fields = ExprNodeDescUtils.extractFields(fieldDesc);
}
// We also need to update the expr so that the index query can be
// generated.
// Note that, hive does not support UDFToDouble etc in the query text.
List<ExprNodeDesc> list = new ArrayList<ExprNodeDesc>();
list.add(expr1);
list.add(expr2);
expr = new ExprNodeGenericFuncDesc(expr.getTypeInfo(), expr.getGenericUDF(), list);
searchConditions.add(new IndexSearchCondition(columnDesc, udfName, constantDesc, expr, fields));
// remove it from the residual predicate
return fields == null ? null : expr;
}
use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull in project hive by apache.
the class ConstantPropagateProcFactory method shortcutFunction.
private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs, Operator<? extends Serializable> op) throws UDFArgumentException {
if (udf instanceof GenericUDFOPEqual) {
assert newExprs.size() == 2;
boolean foundUDFInFirst = false;
ExprNodeGenericFuncDesc caseOrWhenexpr = null;
if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) {
caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0);
if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) {
foundUDFInFirst = true;
}
}
if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) {
caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1);
if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) {
return null;
}
}
if (null == caseOrWhenexpr) {
// we didn't find case or when udf
return null;
}
GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
List<ExprNodeDesc> children = caseOrWhenexpr.getChildren();
int i;
if (childUDF instanceof GenericUDFWhen) {
for (i = 1; i < children.size(); i += 2) {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
}
if (children.size() % 2 == 1) {
i = children.size() - 1;
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
}
// after constant folding of child expression the return type of UDFWhen might have changed,
// so recreate the expression
ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
return newCaseOrWhenExpr;
} else if (childUDF instanceof GenericUDFCase) {
for (i = 2; i < children.size(); i += 2) {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
}
if (children.size() % 2 == 0) {
i = children.size() - 1;
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
}
// after constant folding of child expression the return type of UDFCase might have changed,
// so recreate the expression
ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
return newCaseOrWhenExpr;
} else {
// cant happen
return null;
}
}
if (udf instanceof GenericUDFOPAnd) {
final BitSet positionsToRemove = new BitSet();
final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
final List<Integer> notNullExprsPositions = new ArrayList<Integer>();
final List<ExprNodeDesc> compareExprs = new ArrayList<ExprNodeDesc>();
for (int i = 0; i < newExprs.size(); i++) {
ExprNodeDesc childExpr = newExprs.get(i);
if (childExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
if (Boolean.TRUE.equals(c.getValue())) {
// if true, prune it
positionsToRemove.set(i);
} else {
// if false, return false
return childExpr;
}
} else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
notNullExprs.add(childExpr.getChildren().get(0));
notNullExprsPositions.add(i);
} else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFBaseCompare && !(((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) && childExpr.getChildren().size() == 2) {
// Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
// where <op> can be "=", ">=", "<=", ">", "<".
// Note: (key <> 86) and (key is not null) cannot be folded
ExprNodeColumnDesc colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(0));
if (null == colDesc) {
colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(1));
}
if (colDesc != null) {
compareExprs.add(colDesc);
}
}
}
// Try to fold (key = 86) and (key is not null) to (key = 86)
for (int i = 0; i < notNullExprs.size(); i++) {
for (ExprNodeDesc other : compareExprs) {
if (notNullExprs.get(i).isSame(other)) {
positionsToRemove.set(notNullExprsPositions.get(i));
break;
}
}
}
// Remove unnecessary expressions
int pos = 0;
int removed = 0;
while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
newExprs.remove(pos - removed);
pos++;
removed++;
}
if (newExprs.size() == 0) {
return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
}
if (newExprs.size() == 1) {
return newExprs.get(0);
}
}
if (udf instanceof GenericUDFOPOr) {
final BitSet positionsToRemove = new BitSet();
for (int i = 0; i < newExprs.size(); i++) {
ExprNodeDesc childExpr = newExprs.get(i);
if (childExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
if (Boolean.FALSE.equals(c.getValue())) {
// if false, prune it
positionsToRemove.set(i);
} else if (Boolean.TRUE.equals(c.getValue())) {
// if true return true
return childExpr;
}
}
}
int pos = 0;
int removed = 0;
while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
newExprs.remove(pos - removed);
pos++;
removed++;
}
if (newExprs.size() == 0) {
return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.FALSE);
}
if (newExprs.size() == 1) {
return newExprs.get(0);
}
}
if (udf instanceof GenericUDFWhen) {
if (!(newExprs.size() == 2 || newExprs.size() == 3)) {
// we currently only handle either 1 or 2 branch.
return null;
}
ExprNodeDesc thenExpr = newExprs.get(1);
ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) : new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(), null);
ExprNodeDesc whenExpr = newExprs.get(0);
if (whenExpr instanceof ExprNodeConstantDesc) {
Boolean whenVal = (Boolean) ((ExprNodeConstantDesc) whenExpr).getValue();
return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr;
}
if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
Object thenVal = constThen.getValue();
Object elseVal = constElse.getValue();
if (thenVal == null) {
if (elseVal == null) {
// both branches are null.
return thenExpr;
} else if (op instanceof FilterOperator) {
// we can still fold, since here null is equivalent to false.
return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
} else {
// can't do much, expression is not in context of filter, so we can't treat null as equivalent to false here.
return null;
}
} else if (elseVal == null && op instanceof FilterOperator) {
return Boolean.TRUE.equals(thenVal) ? whenExpr : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
} else if (thenVal.equals(elseVal)) {
return thenExpr;
} else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
List<ExprNodeDesc> children = new ArrayList<>();
children.add(whenExpr);
children.add(new ExprNodeConstantDesc(false));
ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children);
if (Boolean.TRUE.equals(thenVal)) {
return func;
} else {
List<ExprNodeDesc> exprs = new ArrayList<>();
exprs.add(func);
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
}
} else {
return null;
}
}
}
if (udf instanceof GenericUDFCase) {
// where ss_sold_date= '1998-01-01' ;
if (!(newExprs.size() == 3 || newExprs.size() == 4)) {
// we currently only handle either 1 or 2 branch.
return null;
}
ExprNodeDesc thenExpr = newExprs.get(2);
ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) : new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(), null);
if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
Object thenVal = constThen.getValue();
Object elseVal = constElse.getValue();
if (thenVal == null) {
if (null == elseVal) {
return thenExpr;
} else if (op instanceof FilterOperator) {
return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
} else {
return null;
}
} else if (null == elseVal && op instanceof FilterOperator) {
return Boolean.TRUE.equals(thenVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
} else if (thenVal.equals(elseVal)) {
return thenExpr;
} else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
ExprNodeGenericFuncDesc equal = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2));
List<ExprNodeDesc> children = new ArrayList<>();
children.add(equal);
children.add(new ExprNodeConstantDesc(false));
ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children);
if (Boolean.TRUE.equals(thenVal)) {
return func;
} else {
List<ExprNodeDesc> exprs = new ArrayList<>();
exprs.add(func);
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
}
} else {
return null;
}
}
}
if (udf instanceof GenericUDFUnixTimeStamp) {
if (newExprs.size() >= 1) {
// unix_timestamp(args) -> to_unix_timestamp(args)
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), newExprs);
}
}
return null;
}
use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull in project hive by apache.
the class TestVectorizationContext method testNotNullExpressions.
@Test
public void testNotNullExpressions() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(10));
GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
ExprNodeGenericFuncDesc greaterExprDesc = new ExprNodeGenericFuncDesc();
greaterExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
greaterExprDesc.setGenericUDF(udf);
List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeGenericFuncDesc isNotNullExpr = new ExprNodeGenericFuncDesc();
isNotNullExpr.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
GenericUDFOPNotNull notNullUdf = new GenericUDFOPNotNull();
isNotNullExpr.setGenericUDF(notNullUdf);
List<ExprNodeDesc> childOfNot = new ArrayList<ExprNodeDesc>();
childOfNot.add(greaterExprDesc);
isNotNullExpr.setChildren(childOfNot);
List<String> columns = new ArrayList<String>();
columns.add("col1");
columns.add("col2");
VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(isNotNullExpr, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(), SelectColumnIsNotNull.class);
assertEquals(2, ((SelectColumnIsNotNull) ve).getColNum());
assertEquals(ve.getChildExpressions()[0].getClass(), LongColGreaterLongScalar.class);
ve = vc.getVectorExpression(isNotNullExpr, VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(), IsNotNull.class);
assertEquals(2, ((IsNotNull) ve).getColNum());
assertEquals(ve.getChildExpressions()[0].getClass(), LongColGreaterLongScalar.class);
}
use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull in project hive by apache.
the class TestAccumuloPredicateHandler method testPushdownComparisonOptNotSupported.
@Test
public void testPushdownComparisonOptNotSupported() {
try {
ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "field1", null, false);
List<ExprNodeDesc> children = Lists.newArrayList();
children.add(column);
ExprNodeGenericFuncDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo, new GenericUDFOPNotNull(), children);
assertNotNull(node);
String filterExpr = SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, filterExpr);
List<IndexSearchCondition> sConditions = handler.getSearchConditions(conf);
assertEquals(sConditions.size(), 1);
IndexSearchCondition sc = sConditions.get(0);
new PushdownTuple(sc, handler.getPrimitiveComparison(sc.getColumnDesc().getTypeString(), sc), handler.getCompareOp(sc.getComparisonOp(), sc));
fail("Should fail: compare op not registered for index analyzer. Should leave undesirable residual predicate");
} catch (RuntimeException e) {
assertTrue(e.getMessage().contains("Unexpected residual predicate: field1 is not null"));
} catch (Exception e) {
fail(StringUtils.stringifyException(e));
}
}
Aggregations