Examples with GenericUDFCoalesce org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce used on opensource projects

Example 1 with GenericUDFCoalesce

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce in project hive by apache.

the class TestVectorCoalesceElt method doCoalesceOnRandomDataType.

private boolean doCoalesceOnRandomDataType(Random random, int iteration, boolean isCoalesce, boolean isEltIndexConst, int columnCount, int[] constantColumns, int[] nullConstantColumns, boolean allowNulls) throws Exception {
    String typeName;
    if (isCoalesce) {
        typeName = VectorRandomRowSource.getRandomTypeName(random, SupportedTypes.PRIMITIVES, /* allowedTypeNameSet */
        null);
        typeName = VectorRandomRowSource.getDecoratedTypeName(random, typeName, SupportedTypes.PRIMITIVES, /* allowedTypeNameSet */
        null, /* depth */
        0, /* maxDepth */
        2);
    } else {
        // ELT only choose between STRINGs.
        typeName = "string";
    }
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    // ----------------------------------------------------------------------------------------------
    final TypeInfo intTypeInfo;
    ObjectInspector intObjectInspector;
    if (isCoalesce) {
        intTypeInfo = null;
        intObjectInspector = null;
    } else {
        intTypeInfo = TypeInfoFactory.intTypeInfo;
        intObjectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(intTypeInfo);
    }
    ObjectInspector objectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
    // ----------------------------------------------------------------------------------------------
    List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    List<String> columns = new ArrayList<String>();
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    int columnNum = 1;
    if (!isCoalesce) {
        List<Object> intValueList = new ArrayList<Object>();
        for (int i = -1; i < columnCount + 2; i++) {
            intValueList.add(new IntWritable(i));
        }
        final int intValueListCount = intValueList.size();
        ExprNodeDesc intColExpr;
        if (!isEltIndexConst) {
            generationSpecList.add(GenerationSpec.createValueList(intTypeInfo, intValueList));
            explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
            String columnName = "col" + columnNum++;
            columns.add(columnName);
            intColExpr = new ExprNodeColumnDesc(intTypeInfo, columnName, "table", false);
        } else {
            final Object scalarObject;
            if (random.nextInt(10) != 0) {
                scalarObject = intValueList.get(random.nextInt(intValueListCount));
            } else {
                scalarObject = null;
            }
            intColExpr = new ExprNodeConstantDesc(typeInfo, scalarObject);
        }
        children.add(intColExpr);
    }
    for (int c = 0; c < columnCount; c++) {
        ExprNodeDesc colExpr;
        if (!contains(constantColumns, c)) {
            generationSpecList.add(GenerationSpec.createSameType(typeInfo));
            explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
            String columnName = "col" + columnNum++;
            columns.add(columnName);
            colExpr = new ExprNodeColumnDesc(typeInfo, columnName, "table", false);
        } else {
            final Object scalarObject;
            if (!contains(nullConstantColumns, c)) {
                scalarObject = VectorRandomRowSource.randomPrimitiveObject(random, (PrimitiveTypeInfo) typeInfo);
            } else {
                scalarObject = null;
            }
            colExpr = new ExprNodeConstantDesc(typeInfo, scalarObject);
        }
        children.add(colExpr);
    }
    VectorRandomRowSource rowSource = new VectorRandomRowSource();
    rowSource.initGenerationSpecSchema(random, generationSpecList, /* maxComplexDepth */
    0, /* allowNull */
    allowNulls, /* isUnicodeOk */
    true, explicitDataTypePhysicalVariationList);
    String[] columnNames = columns.toArray(new String[0]);
    Object[][] randomRows = rowSource.randomRows(100000);
    VectorRandomBatchSource batchSource = VectorRandomBatchSource.createInterestingBatches(random, rowSource, randomRows, null);
    final GenericUDF udf = (isCoalesce ? new GenericUDFCoalesce() : new GenericUDFElt());
    final int start = isCoalesce ? 0 : 1;
    final int end = start + columnCount;
    ObjectInspector[] argumentOIs = new ObjectInspector[end];
    if (!isCoalesce) {
        argumentOIs[0] = intObjectInspector;
    }
    for (int i = start; i < end; i++) {
        argumentOIs[i] = objectInspector;
    }
    final ObjectInspector outputObjectInspector = udf.initialize(argumentOIs);
    TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(typeInfo, udf, children);
    final int rowCount = randomRows.length;
    Object[][] resultObjectsArray = new Object[CoalesceEltTestMode.count][];
    for (int i = 0; i < CoalesceEltTestMode.count; i++) {
        Object[] resultObjects = new Object[rowCount];
        resultObjectsArray[i] = resultObjects;
        CoalesceEltTestMode coalesceEltTestMode = CoalesceEltTestMode.values()[i];
        switch(coalesceEltTestMode) {
            case ROW_MODE:
                if (!doRowCastTest(typeInfo, columns, children, udf, exprDesc, randomRows, rowSource.rowStructObjectInspector(), exprDesc.getWritableObjectInspector(), resultObjects)) {
                    return false;
                }
                break;
            case ADAPTOR:
            case VECTOR_EXPRESSION:
                if (!doVectorCastTest(typeInfo, iteration, columns, columnNames, rowSource.typeInfos(), rowSource.dataTypePhysicalVariations(), children, udf, exprDesc, coalesceEltTestMode, batchSource, exprDesc.getWritableObjectInspector(), outputTypeInfo, resultObjects)) {
                    return false;
                }
                break;
            default:
                throw new RuntimeException("Unexpected IF statement test mode " + coalesceEltTestMode);
        }
    }
    for (int i = 0; i < rowCount; i++) {
        // Row-mode is the expected value.
        Object expectedResult = resultObjectsArray[0][i];
        for (int v = 1; v < CoalesceEltTestMode.count; v++) {
            Object vectorResult = resultObjectsArray[v][i];
            CoalesceEltTestMode coalesceEltTestMode = CoalesceEltTestMode.values()[v];
            if (expectedResult == null || vectorResult == null) {
                if (expectedResult != null || vectorResult != null) {
                    Assert.fail("Row " + i + " sourceTypeName " + typeName + " " + coalesceEltTestMode + " iteration " + iteration + " result is NULL " + (vectorResult == null ? "YES" : "NO result " + vectorResult.toString()) + " does not match row-mode expected result is NULL " + (expectedResult == null ? "YES" : "NO result '" + expectedResult.toString()) + "'" + " row values " + Arrays.toString(randomRows[i]) + " exprDesc " + exprDesc.toString());
                }
            } else {
                if (!expectedResult.equals(vectorResult)) {
                    Assert.fail("Row " + i + " sourceTypeName " + typeName + " " + coalesceEltTestMode + " iteration " + iteration + " result '" + vectorResult.toString() + "'" + " (" + vectorResult.getClass().getSimpleName() + ")" + " does not match row-mode expected result '" + expectedResult.toString() + "'" + " (" + expectedResult.getClass().getSimpleName() + ")" + " row values " + Arrays.toString(randomRows[i]) + " exprDesc " + exprDesc.toString());
                }
            }
        }
    }
    return true;
}

Example 2 with GenericUDFCoalesce

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce in project hive by apache.

the class ConstantPropagateProcFactory method shortcutFunction.

private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs, Operator<? extends Serializable> op) throws UDFArgumentException {
    if (udf instanceof GenericUDFOPEqual) {
        assert newExprs.size() == 2;
        boolean foundUDFInFirst = false;
        ExprNodeGenericFuncDesc caseOrWhenexpr = null;
        if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) {
            caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0);
            if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) {
                foundUDFInFirst = true;
            }
        }
        if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) {
            caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1);
            if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) {
                return null;
            }
        }
        if (null == caseOrWhenexpr) {
            // we didn't find case or when udf
            return null;
        }
        GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
        List<ExprNodeDesc> children = new ArrayList(caseOrWhenexpr.getChildren());
        int i;
        if (childUDF instanceof GenericUDFWhen) {
            for (i = 1; i < children.size(); i += 2) {
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            if (children.size() % 2 == 1) {
                i = children.size() - 1;
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            // after constant folding of child expression the return type of UDFWhen might have changed,
            // so recreate the expression
            ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
            return newCaseOrWhenExpr;
        } else if (childUDF instanceof GenericUDFCase) {
            for (i = 2; i < children.size(); i += 2) {
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            if (children.size() % 2 == 0) {
                i = children.size() - 1;
                children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0))));
            }
            // after constant folding of child expression the return type of UDFCase might have changed,
            // so recreate the expression
            ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children);
            return newCaseOrWhenExpr;
        } else {
            // cant happen
            return null;
        }
    }
    if (udf instanceof GenericUDFOPAnd) {
        final BitSet positionsToRemove = new BitSet();
        final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
        final List<Integer> notNullExprsPositions = new ArrayList<Integer>();
        final List<ExprNodeDesc> compareExprs = new ArrayList<ExprNodeDesc>();
        for (int i = 0; i < newExprs.size(); i++) {
            ExprNodeDesc childExpr = newExprs.get(i);
            if (childExpr instanceof ExprNodeConstantDesc) {
                ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
                if (Boolean.TRUE.equals(c.getValue())) {
                    // if true, prune it
                    positionsToRemove.set(i);
                } else {
                    if (Boolean.FALSE.equals(c.getValue())) {
                        // if false, return false
                        return childExpr;
                    }
                }
            } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
                notNullExprs.add(childExpr.getChildren().get(0));
                notNullExprsPositions.add(i);
            } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFBaseCompare && !(((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) && childExpr.getChildren().size() == 2) {
                // Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
                // where <op> can be "=", ">=", "<=", ">", "<".
                // Note: (key <> 86) and (key is not null) cannot be folded
                ExprNodeColumnDesc colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(0));
                if (null == colDesc) {
                    colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(1));
                }
                if (colDesc != null) {
                    compareExprs.add(colDesc);
                }
            }
        }
        // Try to fold (key = 86) and (key is not null) to (key = 86)
        for (int i = 0; i < notNullExprs.size(); i++) {
            for (ExprNodeDesc other : compareExprs) {
                if (notNullExprs.get(i).isSame(other)) {
                    positionsToRemove.set(notNullExprsPositions.get(i));
                    break;
                }
            }
        }
        // Remove unnecessary expressions
        int pos = 0;
        int removed = 0;
        while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
            newExprs.remove(pos - removed);
            pos++;
            removed++;
        }
        if (newExprs.size() == 0) {
            return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
        }
        if (newExprs.size() == 1) {
            return newExprs.get(0);
        }
    }
    if (udf instanceof GenericUDFOPOr) {
        final BitSet positionsToRemove = new BitSet();
        for (int i = 0; i < newExprs.size(); i++) {
            ExprNodeDesc childExpr = newExprs.get(i);
            if (childExpr instanceof ExprNodeConstantDesc) {
                ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
                if (Boolean.FALSE.equals(c.getValue())) {
                    // if false, prune it
                    positionsToRemove.set(i);
                } else if (Boolean.TRUE.equals(c.getValue())) {
                    // if true return true
                    return childExpr;
                }
            }
        }
        int pos = 0;
        int removed = 0;
        while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
            newExprs.remove(pos - removed);
            pos++;
            removed++;
        }
        if (newExprs.size() == 0) {
            return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.FALSE);
        }
        if (newExprs.size() == 1) {
            return newExprs.get(0);
        }
    }
    if (udf instanceof GenericUDFWhen) {
        if (!(newExprs.size() == 2 || newExprs.size() == 3)) {
            // we currently only handle either 1 or 2 branch.
            return null;
        }
        ExprNodeDesc thenExpr = newExprs.get(1);
        ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) : new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(), null);
        ExprNodeDesc whenExpr = newExprs.get(0);
        if (whenExpr instanceof ExprNodeConstantDesc) {
            Boolean whenVal = (Boolean) ((ExprNodeConstantDesc) whenExpr).getValue();
            return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr;
        }
        if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
            ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
            Object thenVal = constThen.getValue();
            Object elseVal = constElse.getValue();
            if (thenVal == null) {
                if (elseVal == null) {
                    // both branches are null.
                    return thenExpr;
                } else if (op instanceof FilterOperator) {
                    // we can still fold, since here null is equivalent to false.
                    return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
                } else {
                    // can't do much, expression is not in context of filter, so we can't treat null as equivalent to false here.
                    return null;
                }
            } else if (elseVal == null && op instanceof FilterOperator) {
                return Boolean.TRUE.equals(thenVal) ? whenExpr : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
            } else if (thenVal.equals(elseVal)) {
                return thenExpr;
            } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
                List<ExprNodeDesc> children = new ArrayList<>();
                children.add(whenExpr);
                children.add(new ExprNodeConstantDesc(false));
                ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFCoalesce(), children);
                if (Boolean.TRUE.equals(thenVal)) {
                    return func;
                } else {
                    List<ExprNodeDesc> exprs = new ArrayList<>();
                    exprs.add(func);
                    return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
                }
            } else {
                return null;
            }
        }
    }
    if (udf instanceof GenericUDFCase) {
        // where ss_sold_date= '1998-01-01' ;
        if (!(newExprs.size() == 3 || newExprs.size() == 4)) {
            // we currently only handle either 1 or 2 branch.
            return null;
        }
        ExprNodeDesc thenExpr = newExprs.get(2);
        ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) : new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(), null);
        if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
            ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
            ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
            Object thenVal = constThen.getValue();
            Object elseVal = constElse.getValue();
            if (thenVal == null) {
                if (null == elseVal) {
                    return thenExpr;
                } else if (op instanceof FilterOperator) {
                    return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null;
                } else {
                    return null;
                }
            } else if (null == elseVal && op instanceof FilterOperator) {
                return Boolean.TRUE.equals(thenVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
            } else if (thenVal.equals(elseVal)) {
                return thenExpr;
            } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
                ExprNodeGenericFuncDesc equal = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2));
                List<ExprNodeDesc> children = new ArrayList<>();
                children.add(equal);
                children.add(new ExprNodeConstantDesc(false));
                ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFCoalesce(), children);
                if (Boolean.TRUE.equals(thenVal)) {
                    return func;
                } else {
                    List<ExprNodeDesc> exprs = new ArrayList<>();
                    exprs.add(func);
                    return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
                }
            } else {
                return null;
            }
        }
    }
    if (udf instanceof GenericUDFUnixTimeStamp) {
        if (newExprs.size() >= 1) {
            // unix_timestamp(args) -> to_unix_timestamp(args)
            return ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), newExprs);
        }
    }
    return null;
}