Examples with ExprNodeGenericFuncDesc org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc used on opensource projects

Example 96 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testArithmeticExpressionVectorization.

@Test
public void testArithmeticExpressionVectorization() throws HiveException {
    /**
     * Create original expression tree for following
     * (plus (minus (plus col1 col2) col3) (multiply col4 (mod col5 col6)) )
     */
    GenericUDFOPPlus udf1 = new GenericUDFOPPlus();
    GenericUDFOPMinus udf2 = new GenericUDFOPMinus();
    GenericUDFOPMultiply udf3 = new GenericUDFOPMultiply();
    GenericUDFOPPlus udf4 = new GenericUDFOPPlus();
    GenericUDFOPMod udf5 = new GenericUDFOPMod();
    ExprNodeGenericFuncDesc sumExpr = new ExprNodeGenericFuncDesc();
    sumExpr.setTypeInfo(TypeInfoFactory.intTypeInfo);
    sumExpr.setGenericUDF(udf1);
    ExprNodeGenericFuncDesc minusExpr = new ExprNodeGenericFuncDesc();
    minusExpr.setTypeInfo(TypeInfoFactory.intTypeInfo);
    minusExpr.setGenericUDF(udf2);
    ExprNodeGenericFuncDesc multiplyExpr = new ExprNodeGenericFuncDesc();
    multiplyExpr.setTypeInfo(TypeInfoFactory.intTypeInfo);
    multiplyExpr.setGenericUDF(udf3);
    ExprNodeGenericFuncDesc sum2Expr = new ExprNodeGenericFuncDesc();
    sum2Expr.setTypeInfo(TypeInfoFactory.intTypeInfo);
    sum2Expr.setGenericUDF(udf4);
    ExprNodeGenericFuncDesc modExpr = new ExprNodeGenericFuncDesc();
    modExpr.setTypeInfo(TypeInfoFactory.intTypeInfo);
    modExpr.setGenericUDF(udf5);
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Long.class, "col2", "table", false);
    ExprNodeColumnDesc col3Expr = new ExprNodeColumnDesc(Long.class, "col3", "table", false);
    ExprNodeColumnDesc col4Expr = new ExprNodeColumnDesc(Long.class, "col4", "table", false);
    ExprNodeColumnDesc col5Expr = new ExprNodeColumnDesc(Long.class, "col5", "table", false);
    ExprNodeColumnDesc col6Expr = new ExprNodeColumnDesc(Long.class, "col6", "table", false);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>(2);
    List<ExprNodeDesc> children3 = new ArrayList<ExprNodeDesc>(2);
    List<ExprNodeDesc> children4 = new ArrayList<ExprNodeDesc>(2);
    List<ExprNodeDesc> children5 = new ArrayList<ExprNodeDesc>(2);
    children1.add(minusExpr);
    children1.add(multiplyExpr);
    sumExpr.setChildren(children1);
    children2.add(sum2Expr);
    children2.add(col3Expr);
    minusExpr.setChildren(children2);
    children3.add(col1Expr);
    children3.add(col2Expr);
    sum2Expr.setChildren(children3);
    children4.add(col4Expr);
    children4.add(modExpr);
    multiplyExpr.setChildren(children4);
    children5.add(col5Expr);
    children5.add(col6Expr);
    modExpr.setChildren(children5);
    VectorizationContext vc = new VectorizationContext("name");
    vc.addInitialColumn("col1");
    vc.addInitialColumn("col2");
    vc.addInitialColumn("col3");
    vc.addInitialColumn("col4");
    vc.addInitialColumn("col5");
    vc.addInitialColumn("col6");
    vc.finishedAddingInitialColumns();
    // Generate vectorized expression
    VectorExpression ve = vc.getVectorExpression(sumExpr, VectorExpressionDescriptor.Mode.PROJECTION);
    // Verify vectorized expression
    assertTrue(ve instanceof LongColAddLongColumn);
    assertEquals(2, ve.getChildExpressions().length);
    VectorExpression childExpr1 = ve.getChildExpressions()[0];
    VectorExpression childExpr2 = ve.getChildExpressions()[1];
    System.out.println(ve.toString());
    // TODO: HIVE-20985 disabled output column reuse
    // assertEquals(6, ve.getOutputColumnNum());
    assertEquals(10, ve.getOutputColumnNum());
    assertTrue(childExpr1 instanceof LongColSubtractLongColumn);
    assertEquals(1, childExpr1.getChildExpressions().length);
    assertTrue(childExpr1.getChildExpressions()[0] instanceof LongColAddLongColumn);
    assertEquals(7, childExpr1.getOutputColumnNum());
    assertEquals(6, childExpr1.getChildExpressions()[0].getOutputColumnNum());
    assertTrue(childExpr2 instanceof LongColMultiplyLongColumn);
    assertEquals(1, childExpr2.getChildExpressions().length);
    assertTrue(childExpr2.getChildExpressions()[0] instanceof LongColModuloLongColumn);
    // TODO: HIVE-20985 disabled output column reuse
    // assertEquals(8, childExpr2.getOutputColumnNum());
    // assertEquals(6, childExpr2.getChildExpressions()[0].getOutputColumnNum());
    assertEquals(9, childExpr2.getOutputColumnNum());
    assertEquals(8, childExpr2.getChildExpressions()[0].getOutputColumnNum());
}

Example 97 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testMathFunctions.

@Test
public void testMathFunctions() throws HiveException {
    ExprNodeGenericFuncDesc mathFuncExpr = new ExprNodeGenericFuncDesc();
    mathFuncExpr.setTypeInfo(TypeInfoFactory.doubleTypeInfo);
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Integer.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Double.class, "b", "table", false);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>();
    children1.add(colDesc1);
    children2.add(colDesc2);
    List<String> columns = new ArrayList<String>();
    columns.add("b");
    columns.add("a");
    VectorizationContext vc = new VectorizationContext("name", columns);
    // Sin(double)
    GenericUDFBridge gudfBridge = new GenericUDFBridge("sin", false, UDFSin.class.getName());
    mathFuncExpr.setGenericUDF(gudfBridge);
    mathFuncExpr.setChildren(children2);
    VectorExpression ve = vc.getVectorExpression(mathFuncExpr, VectorExpressionDescriptor.Mode.PROJECTION);
    Assert.assertEquals(FuncSinDoubleToDouble.class, ve.getClass());
    // Round without digits
    GenericUDFRound udfRound = new GenericUDFRound();
    mathFuncExpr.setGenericUDF(udfRound);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
    // BRound without digits
    GenericUDFBRound udfBRound = new GenericUDFBRound();
    mathFuncExpr.setGenericUDF(udfBRound);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncBRoundDoubleToDouble.class, ve.getClass());
    // Round with digits
    mathFuncExpr.setGenericUDF(udfRound);
    children2.add(new ExprNodeConstantDesc(4));
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(RoundWithNumDigitsDoubleToDouble.class, ve.getClass());
    Assert.assertEquals(4, ((RoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
    // BRound with digits
    mathFuncExpr.setGenericUDF(udfBRound);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(BRoundWithNumDigitsDoubleToDouble.class, ve.getClass());
    Assert.assertEquals(4, ((BRoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
    // Logger with int base
    gudfBridge = new GenericUDFBridge("log", false, UDFLog.class.getName());
    mathFuncExpr.setGenericUDF(gudfBridge);
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.0));
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
    // Logger with default base
    children2.clear();
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLnDoubleToDouble.class, ve.getClass());
    // Log with double base
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.5));
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
    // Log with int input and double base
    children2.clear();
    children2.add(new ExprNodeConstantDesc(4.5));
    children2.add(colDesc1);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncLogWithBaseLongToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncLogWithBaseLongToDouble) ve).getBase());
    // Power with double power
    children2.clear();
    children2.add(colDesc2);
    children2.add(new ExprNodeConstantDesc(4.5));
    mathFuncExpr.setGenericUDF(new GenericUDFPower());
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncPowerDoubleToDouble.class, ve.getClass());
    Assert.assertTrue(4.5 == ((FuncPowerDoubleToDouble) ve).getPower());
    // Round with default decimal places
    mathFuncExpr.setGenericUDF(udfRound);
    children2.clear();
    children2.add(colDesc2);
    mathFuncExpr.setChildren(children2);
    ve = vc.getVectorExpression(mathFuncExpr);
    Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
}

Example 98 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testFilterStringColCompareStringColumnExpressions.

@Test
public void testFilterStringColCompareStringColumnExpressions() throws HiveException {
    // Strings test
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false);
    GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // 2 CHAR test
    CharTypeInfo charTypeInfo = new CharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // 2 VARCHAR test
    VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // Some mix tests (STRING, CHAR), (VARCHAR, CHAR), (VARCHAR, STRING)...
    col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
}

Example 99 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testFilterBooleanColumnCompareBooleanScalar.

@Test
public void testFilterBooleanColumnCompareBooleanScalar() throws HiveException {
    ExprNodeGenericFuncDesc colEqualScalar = new ExprNodeGenericFuncDesc();
    GenericUDFOPEqual gudf = new GenericUDFOPEqual();
    colEqualScalar.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, 20);
    ExprNodeColumnDesc colDesc = new ExprNodeColumnDesc(Boolean.class, "a", "table", false);
    children.add(colDesc);
    children.add(constDesc);
    colEqualScalar.setChildren(children);
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterLongColEqualLongScalar.class, ve.getClass());
}

Example 100 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testUnaryMinusColumnDouble.

@Test
public void testUnaryMinusColumnDouble() throws HiveException {
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Float.class, "col1", "table", false);
    GenericUDF gudf = new GenericUDFOPNegative();
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
    children.add(col1Expr);
    ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.doubleTypeInfo, gudf, children);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
    assertTrue(ve instanceof DoubleColUnaryMinus);
}