use of org.apache.hadoop.hive.ql.CompilationOpContext in project hive by apache.
the class TestVectorFilterOperator method getAVectorFilterOperator.
private VectorFilterOperator getAVectorFilterOperator() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false);
List<String> columns = new ArrayList<String>();
columns.add("col1");
FilterDesc fdesc = new FilterDesc();
fdesc.setPredicate(col1Expr);
Operator<? extends OperatorDesc> filterOp = OperatorFactory.get(new CompilationOpContext(), fdesc);
VectorizationContext vc = new VectorizationContext("name", columns);
return (VectorFilterOperator) Vectorizer.vectorizeFilterOperator(filterOp, vc);
}
use of org.apache.hadoop.hive.ql.CompilationOpContext in project hive by apache.
the class TestVectorGroupByOperator method testKeyTypeAggregate.
private void testKeyTypeAggregate(String aggregateName, FakeVectorRowBatchFromObjectIterables data, Map<Object, Object> expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("Key");
mapColumnNames.add("Value");
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
Set<Object> keys = new HashSet<Object>();
AggregationDesc agg = buildAggregationDesc(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "Value", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[1]));
ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
aggs.add(agg);
ArrayList<String> outputColumnNames = new ArrayList<String>();
outputColumnNames.add("_col0");
outputColumnNames.add("_col1");
GroupByDesc desc = new GroupByDesc();
desc.setVectorDesc(new VectorGroupByDesc());
desc.setOutputColumnNames(outputColumnNames);
desc.setAggregators(aggs);
((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH);
ExprNodeDesc keyExp = buildColumnDesc(ctx, "Key", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[0]));
ArrayList<ExprNodeDesc> keysDesc = new ArrayList<ExprNodeDesc>();
keysDesc.add(keyExp);
desc.setKeys(keysDesc);
CompilationOpContext cCtx = new CompilationOpContext();
Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc);
VectorGroupByOperator vgo = (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo);
vgo.initialize(hconf, null);
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
private int rowIndex;
private String aggregateName;
private Map<Object, Object> expected;
private Set<Object> keys;
@Override
public void inspectRow(Object row, int tag) throws HiveException {
assertTrue(row instanceof Object[]);
Object[] fields = (Object[]) row;
assertEquals(2, fields.length);
Object key = fields[0];
Object keyValue = null;
if (null == key) {
keyValue = null;
} else if (key instanceof ByteWritable) {
ByteWritable bwKey = (ByteWritable) key;
keyValue = bwKey.get();
} else if (key instanceof ShortWritable) {
ShortWritable swKey = (ShortWritable) key;
keyValue = swKey.get();
} else if (key instanceof IntWritable) {
IntWritable iwKey = (IntWritable) key;
keyValue = iwKey.get();
} else if (key instanceof LongWritable) {
LongWritable lwKey = (LongWritable) key;
keyValue = lwKey.get();
} else if (key instanceof TimestampWritable) {
TimestampWritable twKey = (TimestampWritable) key;
keyValue = twKey.getTimestamp();
} else if (key instanceof DoubleWritable) {
DoubleWritable dwKey = (DoubleWritable) key;
keyValue = dwKey.get();
} else if (key instanceof FloatWritable) {
FloatWritable fwKey = (FloatWritable) key;
keyValue = fwKey.get();
} else if (key instanceof BooleanWritable) {
BooleanWritable bwKey = (BooleanWritable) key;
keyValue = bwKey.get();
} else if (key instanceof HiveDecimalWritable) {
HiveDecimalWritable hdwKey = (HiveDecimalWritable) key;
keyValue = hdwKey.getHiveDecimal();
} else {
Assert.fail(String.format("Not implemented key output type %s: %s", key.getClass().getName(), key));
}
String keyValueAsString = String.format("%s", keyValue);
assertTrue(expected.containsKey(keyValue));
Object expectedValue = expected.get(keyValue);
Object value = fields[1];
Validator validator = getValidator(aggregateName);
validator.validate(keyValueAsString, expectedValue, new Object[] { value });
keys.add(keyValue);
}
private FakeCaptureOutputOperator.OutputInspector init(String aggregateName, Map<Object, Object> expected, Set<Object> keys) {
this.aggregateName = aggregateName;
this.expected = expected;
this.keys = keys;
return this;
}
}.init(aggregateName, expected, keys));
for (VectorizedRowBatch unit : data) {
vgo.process(unit, 0);
}
vgo.close(false);
List<Object> outBatchList = out.getCapturedRows();
assertNotNull(outBatchList);
assertEquals(expected.size(), outBatchList.size());
assertEquals(expected.size(), keys.size());
}
use of org.apache.hadoop.hive.ql.CompilationOpContext in project hive by apache.
the class FakeVectorDataSourceOperator method addFakeVectorDataSourceParent.
public static FakeVectorDataSourceOperator addFakeVectorDataSourceParent(Iterable<VectorizedRowBatch> source, Operator<? extends OperatorDesc> op) {
FakeVectorDataSourceOperator parent = new FakeVectorDataSourceOperator(new CompilationOpContext(), source);
List<Operator<? extends OperatorDesc>> listParents = new ArrayList<Operator<? extends OperatorDesc>>(1);
listParents.add(parent);
op.setParentOperators(listParents);
List<Operator<? extends OperatorDesc>> listChildren = new ArrayList<Operator<? extends OperatorDesc>>(1);
listChildren.add(op);
parent.setChildOperators(listChildren);
return parent;
}
use of org.apache.hadoop.hive.ql.CompilationOpContext in project hive by apache.
the class TestExecDriver method setUp.
@Override
protected void setUp() {
mr = PlanUtils.getMapRedWork();
ctx = new CompilationOpContext();
}
use of org.apache.hadoop.hive.ql.CompilationOpContext in project hive by apache.
the class TestOperators method testScriptOperatorBlacklistedEnvVarsProcessing.
public void testScriptOperatorBlacklistedEnvVarsProcessing() {
ScriptOperator scriptOperator = new ScriptOperator(new CompilationOpContext());
Configuration hconf = new JobConf(ScriptOperator.class);
Map<String, String> env = new HashMap<String, String>();
HiveConf.setVar(hconf, HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST, "foobar");
hconf.set("foobar", "foobar");
hconf.set("barfoo", "barfoo");
scriptOperator.addJobConfToEnvironment(hconf, env);
Assert.assertFalse(env.containsKey("foobar"));
Assert.assertTrue(env.containsKey("barfoo"));
}
Aggregations