Examples with ExprNodeConstantDesc org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc used on opensource projects

Example 81 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TestVectorizationContext method testMathFunctions.

@Test
public void testMathFunctions() throws HiveException {
    ExprNodeGenericFuncDesc mathFuncExpr = new ExprNodeGenericFuncDesc();
    mathFuncExpr.setTypeInfo(TypeInfoFactory.doubleTypeInfo);
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Integer.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Double.class, "b", "table", false);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>();
    children1.add(colDesc1);
    children2.add(colDesc2);
    List<String> columns = new ArrayList<String>();
    columns.add("b");
    columns.add("a");
    VectorizationContext vc = new VectorizationContext("name", columns);
    // Sin(double)
    GenericUDFBridge gudfBridge = new GenericUDFBridge("sin", false, UDFSin.class.getName());
    mathFuncExpr.setGenericUDF(gudfBridge);
    mathFuncExpr.setChildren(children2);
    VectorExpression ve = vc.getVectorExpression(mathFuncExpr, VectorExpressionDescriptor.Mode.PROJECTION);
    Assert.assertEquals(FuncSinDoubleToDouble.class, ve.getClass());
    // Round without digits
    GenericUDFRound udfRound = new GenericUDFRound();
    mathFuncExpr.setGenericUDF(udfRound);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
    // BRound without digits
    GenericUDFBRound udfBRound = new GenericUDFBRound();
    mathFuncExpr.setGenericUDF(udfBRound);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncBRoundDoubleToDouble.class, ve.getClass());
    // Round with digits
    mathFuncExpr.setGenericUDF(udfRound);
    children2.add(new ExprNodeConstantDesc(4));
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(RoundWithNumDigitsDoubleToDouble.class, ve.getClass());
    Assert.assertEquals(4, ((RoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
    // BRound with digits
    mathFuncExpr.setGenericUDF(udfBRound);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(BRoundWithNumDigitsDoubleToDouble.class, ve.getClass());
    Assert.assertEquals(4, ((BRoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
    // Logger with int base
    gudfBridge = new GenericUDFBridge("log", false, UDFLog.class.getName());
    mathFuncExpr.setGenericUDF(gudfBridge);
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.0));
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
    // Logger with default base
    children2.clear();
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLnDoubleToDouble.class, ve.getClass());
    // Log with double base
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.5));
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
    // Log with int input and double base
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.5));
    children2.add(colDesc1);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseLongToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncLogWithBaseLongToDouble) ve).getBase());
    // Power with double power
    children2.clear();
    children2.add(colDesc2);
    children2.add(new ExprNodeConstantDesc(4.5));
    mathFuncExpr.setGenericUDF(new GenericUDFPower());
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncPowerDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncPowerDoubleToDouble) ve).getPower());
    // Round with default decimal places
    mathFuncExpr.setGenericUDF(udfRound);
    children2.clear();
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
}

Example 82 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TestVectorizationContext method testFilterBooleanColumnCompareBooleanScalar.

@Test
public void testFilterBooleanColumnCompareBooleanScalar() throws HiveException {
    ExprNodeGenericFuncDesc colEqualScalar = new ExprNodeGenericFuncDesc();
    GenericUDFOPEqual gudf = new GenericUDFOPEqual();
    colEqualScalar.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, 20);
    ExprNodeColumnDesc colDesc = new ExprNodeColumnDesc(Boolean.class, "a", "table", false);
    children.add(colDesc);
    children.add(constDesc);
    colEqualScalar.setChildren(children);
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterLongColEqualLongScalar.class, ve.getClass());
}

Example 83 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TestVectorBetweenIn method doBetweenInVariation.

private boolean doBetweenInVariation(Random random, String typeName, boolean tryDecimal64, BetweenInVariation betweenInVariation, int subVariation) throws Exception {
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    boolean isDecimal64 = checkDecimal64(tryDecimal64, typeInfo);
    DataTypePhysicalVariation dataTypePhysicalVariation = (isDecimal64 ? DataTypePhysicalVariation.DECIMAL_64 : DataTypePhysicalVariation.NONE);
    final int decimal64Scale = (isDecimal64 ? ((DecimalTypeInfo) typeInfo).getScale() : 0);
    // ----------------------------------------------------------------------------------------------
    ObjectInspector objectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
    final int valueCount = 10 + random.nextInt(10);
    List<Object> valueList = new ArrayList<Object>(valueCount);
    for (int i = 0; i < valueCount; i++) {
        valueList.add(VectorRandomRowSource.randomWritable(random, typeInfo, objectInspector, dataTypePhysicalVariation, /* allowNull */
        false));
    }
    final boolean isBetween = (betweenInVariation == BetweenInVariation.FILTER_BETWEEN || betweenInVariation == BetweenInVariation.FILTER_NOT_BETWEEN || betweenInVariation == BetweenInVariation.PROJECTION_BETWEEN || betweenInVariation == BetweenInVariation.PROJECTION_NOT_BETWEEN);
    List<Object> compareList = new ArrayList<Object>();
    List<Object> sortedList = new ArrayList<Object>(valueCount);
    sortedList.addAll(valueList);
    Object exampleObject = valueList.get(0);
    WritableComparator writableComparator = WritableComparator.get((Class<? extends WritableComparable>) exampleObject.getClass());
    sortedList.sort(writableComparator);
    final boolean isInvert;
    if (isBetween) {
        // FILTER_BETWEEN
        // FILTER_NOT_BETWEEN
        // PROJECTION_BETWEEN
        // PROJECTION_NOT_BETWEEN
        isInvert = (betweenInVariation == BetweenInVariation.FILTER_NOT_BETWEEN || betweenInVariation == BetweenInVariation.PROJECTION_NOT_BETWEEN);
        switch(subVariation) {
            case 0:
                // Range covers all values exactly.
                compareList.add(sortedList.get(0));
                compareList.add(sortedList.get(valueCount - 1));
                break;
            case 1:
                // Exclude the first and last sorted.
                compareList.add(sortedList.get(1));
                compareList.add(sortedList.get(valueCount - 2));
                break;
            case 2:
                // Only last 2 sorted.
                compareList.add(sortedList.get(valueCount - 2));
                compareList.add(sortedList.get(valueCount - 1));
                break;
            case 3:
            case 4:
            case 5:
            case 6:
                {
                    // Choose 2 adjacent in the middle.
                    Object min = sortedList.get(5);
                    Object max = sortedList.get(6);
                    compareList.add(min);
                    compareList.add(max);
                    if (subVariation == 4) {
                        removeValue(valueList, min);
                    } else if (subVariation == 5) {
                        removeValue(valueList, max);
                    } else if (subVariation == 6) {
                        removeValue(valueList, min);
                        removeValue(valueList, max);
                    }
                }
                break;
            default:
                return false;
        }
    } else {
        // FILTER_IN.
        // PROJECTION_IN.
        isInvert = false;
        switch(subVariation) {
            case 0:
                // All values.
                compareList.addAll(valueList);
                break;
            case 1:
                // Don't include the first and last sorted.
                for (int i = 1; i < valueCount - 1; i++) {
                    compareList.add(valueList.get(i));
                }
                break;
            case 2:
                // The even ones.
                for (int i = 2; i < valueCount; i += 2) {
                    compareList.add(valueList.get(i));
                }
                break;
            case 3:
                {
                    // Choose 2 adjacent in the middle.
                    Object min = sortedList.get(5);
                    Object max = sortedList.get(6);
                    compareList.add(min);
                    compareList.add(max);
                    if (subVariation == 4) {
                        removeValue(valueList, min);
                    } else if (subVariation == 5) {
                        removeValue(valueList, max);
                    } else if (subVariation == 6) {
                        removeValue(valueList, min);
                        removeValue(valueList, max);
                    }
                }
                break;
            default:
                return false;
        }
    }
    // ----------------------------------------------------------------------------------------------
    GenerationSpec generationSpec = GenerationSpec.createValueList(typeInfo, valueList);
    List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    generationSpecList.add(generationSpec);
    explicitDataTypePhysicalVariationList.add(dataTypePhysicalVariation);
    VectorRandomRowSource rowSource = new VectorRandomRowSource();
    rowSource.initGenerationSpecSchema(random, generationSpecList, /* maxComplexDepth */
    0, /* allowNull */
    true, /* isUnicodeOk */
    true, explicitDataTypePhysicalVariationList);
    List<String> columns = new ArrayList<String>();
    String col1Name = rowSource.columnNames().get(0);
    columns.add(col1Name);
    final ExprNodeDesc col1Expr = new ExprNodeColumnDesc(typeInfo, col1Name, "table", false);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    if (isBetween) {
        children.add(new ExprNodeConstantDesc(Boolean.valueOf(isInvert)));
    }
    children.add(col1Expr);
    for (Object compareObject : compareList) {
        ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(typeInfo, VectorRandomRowSource.getNonWritableObject(compareObject, typeInfo, objectInspector));
        children.add(constDesc);
    }
    String[] columnNames = columns.toArray(new String[0]);
    Object[][] randomRows = rowSource.randomRows(100000);
    VectorRandomBatchSource batchSource = VectorRandomBatchSource.createInterestingBatches(random, rowSource, randomRows, null);
    final GenericUDF udf;
    final ObjectInspector outputObjectInspector;
    if (isBetween) {
        udf = new GenericUDFBetween();
        // First argument is boolean invert. Arguments 1..3 are inspectors for range limits...
        ObjectInspector[] argumentOIs = new ObjectInspector[4];
        argumentOIs[0] = PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
        argumentOIs[1] = objectInspector;
        argumentOIs[2] = objectInspector;
        argumentOIs[3] = objectInspector;
        outputObjectInspector = udf.initialize(argumentOIs);
    } else {
        final int compareCount = compareList.size();
        udf = new GenericUDFIn();
        ObjectInspector[] argumentOIs = new ObjectInspector[compareCount];
        ConstantObjectInspector constantObjectInspector = (ConstantObjectInspector) children.get(1).getWritableObjectInspector();
        for (int i = 0; i < compareCount; i++) {
            argumentOIs[i] = constantObjectInspector;
        }
        outputObjectInspector = udf.initialize(argumentOIs);
    }
    TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children);
    return executeTestModesAndVerify(typeInfo, betweenInVariation, compareList, columns, columnNames, children, udf, exprDesc, randomRows, rowSource, batchSource, outputTypeInfo, /* skipAdaptor */
    false);
}

Example 84 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TestVectorBetweenIn method doBetweenStructInVariation.

private boolean doBetweenStructInVariation(Random random, String structTypeName, BetweenInVariation betweenInVariation) throws Exception {
    StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(structTypeName);
    ObjectInspector structObjectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(structTypeInfo);
    final int valueCount = 10 + random.nextInt(10);
    List<Object> valueList = new ArrayList<Object>(valueCount);
    for (int i = 0; i < valueCount; i++) {
        valueList.add(VectorRandomRowSource.randomWritable(random, structTypeInfo, structObjectInspector, DataTypePhysicalVariation.NONE, /* allowNull */
        false));
    }
    final boolean isInvert = false;
    // No convenient WritableComparator / WritableComparable available for STRUCT.
    List<Object> compareList = new ArrayList<Object>();
    Set<Integer> includedSet = new HashSet<Integer>();
    final int chooseLimit = 4 + random.nextInt(valueCount / 2);
    int chooseCount = 0;
    while (chooseCount < chooseLimit) {
        final int index = random.nextInt(valueCount);
        if (includedSet.contains(index)) {
            continue;
        }
        includedSet.add(index);
        compareList.add(valueList.get(index));
        chooseCount++;
    }
    // ----------------------------------------------------------------------------------------------
    GenerationSpec structGenerationSpec = GenerationSpec.createValueList(structTypeInfo, valueList);
    List<GenerationSpec> structGenerationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> structExplicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    structGenerationSpecList.add(structGenerationSpec);
    structExplicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
    VectorRandomRowSource structRowSource = new VectorRandomRowSource();
    structRowSource.initGenerationSpecSchema(random, structGenerationSpecList, /* maxComplexDepth */
    0, /* allowNull */
    true, /* isUnicodeOk */
    true, structExplicitDataTypePhysicalVariationList);
    Object[][] structRandomRows = structRowSource.randomRows(100000);
    // ---------------------------------------------------------------------------------------------
    List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    List<TypeInfo> fieldTypeInfoList = structTypeInfo.getAllStructFieldTypeInfos();
    final int fieldCount = fieldTypeInfoList.size();
    for (int i = 0; i < fieldCount; i++) {
        GenerationSpec generationSpec = GenerationSpec.createOmitGeneration(fieldTypeInfoList.get(i));
        generationSpecList.add(generationSpec);
        explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
    }
    VectorRandomRowSource rowSource = new VectorRandomRowSource();
    rowSource.initGenerationSpecSchema(random, generationSpecList, /* maxComplexDepth */
    0, /* allowNull */
    true, /* isUnicodeOk */
    true, explicitDataTypePhysicalVariationList);
    Object[][] randomRows = rowSource.randomRows(100000);
    final int rowCount = randomRows.length;
    for (int r = 0; r < rowCount; r++) {
        List<Object> fieldValueList = (ArrayList) structRandomRows[r][0];
        for (int f = 0; f < fieldCount; f++) {
            randomRows[r][f] = fieldValueList.get(f);
        }
    }
    // ---------------------------------------------------------------------------------------------
    // Currently, STRUCT IN vectorization assumes a GenericUDFStruct.
    List<ObjectInspector> structUdfObjectInspectorList = new ArrayList<ObjectInspector>();
    List<ExprNodeDesc> structUdfChildren = new ArrayList<ExprNodeDesc>(fieldCount);
    List<String> rowColumnNameList = rowSource.columnNames();
    for (int i = 0; i < fieldCount; i++) {
        TypeInfo fieldTypeInfo = fieldTypeInfoList.get(i);
        ExprNodeColumnDesc fieldExpr = new ExprNodeColumnDesc(fieldTypeInfo, rowColumnNameList.get(i), "table", false);
        structUdfChildren.add(fieldExpr);
        ObjectInspector fieldObjectInspector = VectorRandomRowSource.getObjectInspector(fieldTypeInfo, DataTypePhysicalVariation.NONE);
        structUdfObjectInspectorList.add(fieldObjectInspector);
    }
    StandardStructObjectInspector structUdfObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(rowColumnNameList, structUdfObjectInspectorList);
    String structUdfTypeName = structUdfObjectInspector.getTypeName();
    TypeInfo structUdfTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(structUdfTypeName);
    String structFuncText = "struct";
    FunctionInfo fi = FunctionRegistry.getFunctionInfo(structFuncText);
    GenericUDF genericUDF = fi.getGenericUDF();
    ExprNodeDesc col1Expr = new ExprNodeGenericFuncDesc(structUdfObjectInspector, genericUDF, structFuncText, structUdfChildren);
    // ---------------------------------------------------------------------------------------------
    List<String> columns = new ArrayList<String>();
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    children.add(col1Expr);
    for (int i = 0; i < compareList.size(); i++) {
        Object compareObject = compareList.get(i);
        ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(structUdfTypeInfo, VectorRandomRowSource.getNonWritableObject(compareObject, structUdfTypeInfo, structUdfObjectInspector));
        children.add(constDesc);
    }
    for (int i = 0; i < fieldCount; i++) {
        columns.add(rowColumnNameList.get(i));
    }
    String[] columnNames = columns.toArray(new String[0]);
    VectorRandomBatchSource batchSource = VectorRandomBatchSource.createInterestingBatches(random, rowSource, randomRows, null);
    // ---------------------------------------------------------------------------------------------
    final GenericUDF udf = new GenericUDFIn();
    final int compareCount = compareList.size();
    ObjectInspector[] argumentOIs = new ObjectInspector[compareCount];
    for (int i = 0; i < compareCount; i++) {
        argumentOIs[i] = structUdfObjectInspector;
    }
    final ObjectInspector outputObjectInspector = udf.initialize(argumentOIs);
    TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children);
    return executeTestModesAndVerify(structUdfTypeInfo, betweenInVariation, compareList, columns, columnNames, children, udf, exprDesc, randomRows, rowSource, batchSource, outputTypeInfo, /* skipAdaptor */
    true);
}

Example 85 with ExprNodeConstantDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.

the class TestVectorizationContext method testInFiltersAndExprs.

// Test translation of both IN filters and boolean-valued IN expressions (non-filters).
@Test
public void testInFiltersAndExprs() throws HiveException {
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha");
    ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo");
    // string IN
    GenericUDFIn udf = new GenericUDFIn();
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringColumnInList);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
    assertTrue(ve instanceof StringColumnInList);
    // long IN
    children1.set(0, new ExprNodeColumnDesc(Long.class, "col1", "table", false));
    children1.set(1, new ExprNodeConstantDesc(10));
    children1.set(2, new ExprNodeConstantDesc(20));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterLongColumnInList);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
    assertTrue(ve instanceof LongColumnInList);
    // double IN
    children1.set(0, new ExprNodeColumnDesc(Double.class, "col1", "table", false));
    children1.set(1, new ExprNodeConstantDesc(10d));
    children1.set(2, new ExprNodeConstantDesc(20d));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterDoubleColumnInList);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
    assertTrue(ve instanceof DoubleColumnInList);
}