use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.
the class TestVectorizationContext method testInFiltersAndExprs.
// Test translation of both IN filters and boolean-valued IN expressions (non-filters).
@Test
public void testInFiltersAndExprs() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha");
ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo");
// string IN
GenericUDFIn udf = new GenericUDFIn();
List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
children1.add(col1Expr);
children1.add(constDesc);
children1.add(constDesc2);
ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
List<String> columns = new ArrayList<String>();
columns.add("col0");
columns.add("col1");
columns.add("col2");
VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterStringColumnInList);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
assertTrue(ve instanceof StringColumnInList);
// long IN
children1.set(0, new ExprNodeColumnDesc(Long.class, "col1", "table", false));
children1.set(1, new ExprNodeConstantDesc(10));
children1.set(2, new ExprNodeConstantDesc(20));
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterLongColumnInList);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
assertTrue(ve instanceof LongColumnInList);
// double IN
children1.set(0, new ExprNodeColumnDesc(Double.class, "col1", "table", false));
children1.set(1, new ExprNodeConstantDesc(10d));
children1.set(2, new ExprNodeConstantDesc(20d));
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterDoubleColumnInList);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
assertTrue(ve instanceof DoubleColumnInList);
}
use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.
the class TestVectorizationContext method testVectorizeAndOrProjectionExpression.
@Test
public void testVectorizeAndOrProjectionExpression() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(10));
GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
ExprNodeGenericFuncDesc greaterExprDesc = new ExprNodeGenericFuncDesc();
greaterExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
greaterExprDesc.setGenericUDF(udf);
List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Boolean.class, "col2", "table", false);
GenericUDFOPAnd andUdf = new GenericUDFOPAnd();
ExprNodeGenericFuncDesc andExprDesc = new ExprNodeGenericFuncDesc();
andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
andExprDesc.setGenericUDF(andUdf);
List<ExprNodeDesc> children3 = new ArrayList<ExprNodeDesc>(2);
children3.add(greaterExprDesc);
children3.add(col2Expr);
andExprDesc.setChildren(children3);
List<String> columns = new ArrayList<String>();
columns.add("col1");
columns.add("col2");
VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression veAnd = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veAnd.getClass(), FilterExprAndExpr.class);
assertEquals(veAnd.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
assertEquals(veAnd.getChildExpressions()[1].getClass(), SelectColumnIsTrue.class);
veAnd = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(veAnd.getClass(), ColAndCol.class);
assertEquals(1, veAnd.getChildExpressions().length);
assertEquals(veAnd.getChildExpressions()[0].getClass(), LongColGreaterLongScalar.class);
assertEquals(2, ((ColAndCol) veAnd).getColNum1());
assertEquals(1, ((ColAndCol) veAnd).getColNum2());
assertEquals(3, ((ColAndCol) veAnd).getOutputColumn());
//OR
GenericUDFOPOr orUdf = new GenericUDFOPOr();
ExprNodeGenericFuncDesc orExprDesc = new ExprNodeGenericFuncDesc();
orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
orExprDesc.setGenericUDF(orUdf);
List<ExprNodeDesc> children4 = new ArrayList<ExprNodeDesc>(2);
children4.add(greaterExprDesc);
children4.add(col2Expr);
orExprDesc.setChildren(children4);
//Allocate new Vectorization context to reset the intermediate columns.
vc = new VectorizationContext("name", columns);
VectorExpression veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veOr.getClass(), FilterExprOrExpr.class);
assertEquals(veOr.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
assertEquals(veOr.getChildExpressions()[1].getClass(), SelectColumnIsTrue.class);
veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(veOr.getClass(), ColOrCol.class);
assertEquals(1, veAnd.getChildExpressions().length);
assertEquals(veAnd.getChildExpressions()[0].getClass(), LongColGreaterLongScalar.class);
assertEquals(2, ((ColOrCol) veOr).getColNum1());
assertEquals(1, ((ColOrCol) veOr).getColNum2());
assertEquals(3, ((ColOrCol) veOr).getOutputColumn());
}
use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.
the class TestVectorizationContext method testMathFunctions.
@Test
public void testMathFunctions() throws HiveException {
ExprNodeGenericFuncDesc mathFuncExpr = new ExprNodeGenericFuncDesc();
mathFuncExpr.setTypeInfo(TypeInfoFactory.doubleTypeInfo);
ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Integer.class, "a", "table", false);
ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Double.class, "b", "table", false);
List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>();
children1.add(colDesc1);
children2.add(colDesc2);
List<String> columns = new ArrayList<String>();
columns.add("b");
columns.add("a");
VectorizationContext vc = new VectorizationContext("name", columns);
// Sin(double)
GenericUDFBridge gudfBridge = new GenericUDFBridge("sin", false, UDFSin.class.getName());
mathFuncExpr.setGenericUDF(gudfBridge);
mathFuncExpr.setChildren(children2);
VectorExpression ve = vc.getVectorExpression(mathFuncExpr, VectorExpressionDescriptor.Mode.PROJECTION);
Assert.assertEquals(FuncSinDoubleToDouble.class, ve.getClass());
// Round without digits
GenericUDFRound udfRound = new GenericUDFRound();
mathFuncExpr.setGenericUDF(udfRound);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
// BRound without digits
GenericUDFBRound udfBRound = new GenericUDFBRound();
mathFuncExpr.setGenericUDF(udfBRound);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncBRoundDoubleToDouble.class, ve.getClass());
// Round with digits
mathFuncExpr.setGenericUDF(udfRound);
children2.add(new ExprNodeConstantDesc(4));
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(RoundWithNumDigitsDoubleToDouble.class, ve.getClass());
Assert.assertEquals(4, ((RoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
// BRound with digits
mathFuncExpr.setGenericUDF(udfBRound);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(BRoundWithNumDigitsDoubleToDouble.class, ve.getClass());
Assert.assertEquals(4, ((BRoundWithNumDigitsDoubleToDouble) ve).getDecimalPlaces().get());
// Logger with int base
gudfBridge = new GenericUDFBridge("log", false, UDFLog.class.getName());
mathFuncExpr.setGenericUDF(gudfBridge);
children2.clear();
children2.add(new ExprNodeConstantDesc(4.0));
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
Assert.assertTrue(4 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
// Logger with default base
children2.clear();
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLnDoubleToDouble.class, ve.getClass());
//Log with double base
children2.clear();
children2.add(new ExprNodeConstantDesc(4.5));
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class, ve.getClass());
Assert.assertTrue(4.5 == ((FuncLogWithBaseDoubleToDouble) ve).getBase());
//Log with int input and double base
children2.clear();
children2.add(new ExprNodeConstantDesc(4.5));
children2.add(colDesc1);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseLongToDouble.class, ve.getClass());
Assert.assertTrue(4.5 == ((FuncLogWithBaseLongToDouble) ve).getBase());
//Power with double power
children2.clear();
children2.add(colDesc2);
children2.add(new ExprNodeConstantDesc(4.5));
mathFuncExpr.setGenericUDF(new GenericUDFPower());
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncPowerDoubleToDouble.class, ve.getClass());
Assert.assertTrue(4.5 == ((FuncPowerDoubleToDouble) ve).getPower());
//Round with default decimal places
mathFuncExpr.setGenericUDF(udfRound);
children2.clear();
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncRoundDoubleToDouble.class, ve.getClass());
}
use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.
the class TestUtilities method testSerializeTimestamp.
@Test
public void testSerializeTimestamp() {
Timestamp ts = new Timestamp(1374554702000L);
ts.setNanos(123456);
ExprNodeConstantDesc constant = new ExprNodeConstantDesc(ts);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
children.add(constant);
ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo, new GenericUDFFromUtcTimestamp(), children);
assertEquals(desc.getExprString(), SerializationUtilities.deserializeExpression(SerializationUtilities.serializeExpression(desc)).getExprString());
}
use of org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc in project hive by apache.
the class TestOperators method testScriptOperator.
public void testScriptOperator() throws Throwable {
try {
System.out.println("Testing Script Operator");
// col1
ExprNodeDesc exprDesc1 = TestExecDriver.getStringColumn("col1");
// col2
ExprNodeDesc expr1 = TestExecDriver.getStringColumn("col0");
ExprNodeDesc expr2 = new ExprNodeConstantDesc("1");
ExprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", expr1, expr2);
// select operator to project these two columns
ArrayList<ExprNodeDesc> earr = new ArrayList<ExprNodeDesc>();
earr.add(exprDesc1);
earr.add(exprDesc2);
ArrayList<String> outputCols = new ArrayList<String>();
for (int i = 0; i < earr.size(); i++) {
outputCols.add("_col" + i);
}
SelectDesc selectCtx = new SelectDesc(earr, outputCols);
Operator<SelectDesc> op = OperatorFactory.get(new CompilationOpContext(), SelectDesc.class);
op.setConf(selectCtx);
// scriptOperator to echo the output of the select
TableDesc scriptOutput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
TableDesc scriptInput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
ScriptDesc sd = new ScriptDesc("cat", scriptOutput, TextRecordWriter.class, scriptInput, TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key"));
Operator<ScriptDesc> sop = OperatorFactory.getAndMakeChild(sd, op);
// Collect operator to observe the output of the script
CollectDesc cd = new CollectDesc(Integer.valueOf(10));
CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, sop);
op.initialize(new JobConf(TestOperators.class), new ObjectInspector[] { r[0].oi });
// evaluate on row
for (int i = 0; i < 5; i++) {
op.process(r[i].o, 0);
}
op.close(false);
InspectableObject io = new InspectableObject();
for (int i = 0; i < 5; i++) {
cdop.retrieve(io);
System.out.println("[" + i + "] io.o=" + io.o);
System.out.println("[" + i + "] io.oi=" + io.oi);
StructObjectInspector soi = (StructObjectInspector) io.oi;
assert (soi != null);
StructField a = soi.getStructFieldRef("a");
StructField b = soi.getStructFieldRef("b");
assertEquals("" + (i + 1), ((PrimitiveObjectInspector) a.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, a)));
assertEquals((i) + "1", ((PrimitiveObjectInspector) b.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, b)));
}
System.out.println("Script Operator ok");
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
Aggregations