Search in sources :

Example 1 with VectorSelectOperator

use of org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator in project hive by apache.

the class VectorSelectOperatorBench method setup.

@Setup
public void setup(Blackhole bh) throws HiveException {
    HiveConf hconf = new HiveConf();
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    columns.add("b");
    columns.add("c");
    VectorizationContext vc = new VectorizationContext("name", columns);
    selDesc = new SelectDesc(false);
    List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Long.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Long.class, "b", "table", false);
    ExprNodeColumnDesc colDesc3 = new ExprNodeColumnDesc(Long.class, "c", "table", false);
    ExprNodeGenericFuncDesc plusDesc = new ExprNodeGenericFuncDesc();
    GenericUDF gudf = new GenericUDFOPPlus();
    plusDesc.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    children.add(colDesc1);
    children.add(colDesc2);
    plusDesc.setChildren(children);
    plusDesc.setTypeInfo(TypeInfoFactory.longTypeInfo);
    colList.add(plusDesc);
    colList.add(colDesc3);
    selDesc.setColList(colList);
    List<String> outputColNames = new ArrayList<String>();
    outputColNames.add("_col0");
    outputColNames.add("_col1");
    selDesc.setOutputColumnNames(outputColNames);
    VectorSelectDesc vectorSelectDesc = new VectorSelectDesc();
    selDesc.setVectorDesc(vectorSelectDesc);
    List<ExprNodeDesc> selectColList = selDesc.getColList();
    VectorExpression[] vectorSelectExprs = new VectorExpression[selectColList.size()];
    for (int i = 0; i < selectColList.size(); i++) {
        ExprNodeDesc expr = selectColList.get(i);
        VectorExpression ve = vc.getVectorExpression(expr);
        vectorSelectExprs[i] = ve;
    }
    vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
    vectorSelectDesc.setProjectedOutputColumns(new int[] { 3, 2 });
    CompilationOpContext opContext = new CompilationOpContext();
    vso = new VectorSelectOperator(opContext, selDesc, vc, vectorSelectDesc);
    // to trigger vectorForward
    child = new ArrayList<>();
    child.add(new BlackholeOperator(opContext, bh));
    child.add(new BlackholeOperator(opContext, bh));
    vso.initialize(hconf, null);
    vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(VectorizedRowBatch.DEFAULT_SIZE, 4, 17);
}
Also used : VectorSelectOperator(org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator) ArrayList(java.util.ArrayList) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) VectorizationContext(org.apache.hadoop.hive.ql.exec.vector.VectorizationContext) GenericUDF(org.apache.hadoop.hive.ql.udf.generic.GenericUDF) CompilationOpContext(org.apache.hadoop.hive.ql.CompilationOpContext) BlackholeOperator(org.apache.hive.benchmark.vectorization.BlackholeOperator) GenericUDFOPPlus(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) HiveConf(org.apache.hadoop.hive.conf.HiveConf) VectorSelectDesc(org.apache.hadoop.hive.ql.plan.VectorSelectDesc) VectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression) SelectDesc(org.apache.hadoop.hive.ql.plan.SelectDesc) VectorSelectDesc(org.apache.hadoop.hive.ql.plan.VectorSelectDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Setup(org.openjdk.jmh.annotations.Setup)

Example 2 with VectorSelectOperator

use of org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator in project hive by apache.

the class TestOperators method testSetDoneFromChildOperators.

@SuppressWarnings("unchecked")
@Test
public void testSetDoneFromChildOperators() throws HiveException {
    VectorSelectDesc vectorSelectDesc = new VectorSelectDesc();
    vectorSelectDesc.setProjectedOutputColumns(new int[0]);
    vectorSelectDesc.setSelectExpressions(new VectorExpression[0]);
    VectorSelectOperator selOp = new VectorSelectOperator(new CompilationOpContext(), new SelectDesc(), new VectorizationContext("dummy"), vectorSelectDesc);
    VectorSelectOperator childOp = new VectorSelectOperator(new CompilationOpContext(), new SelectDesc(), new VectorizationContext("dummy"), vectorSelectDesc);
    selOp.childOperatorsArray = new Operator[1];
    selOp.childOperatorsArray[0] = childOp;
    selOp.childOperatorsTag = new int[1];
    selOp.childOperatorsTag[0] = 0;
    childOp.childOperatorsArray = new Operator[0];
    Assert.assertFalse(selOp.getDone());
    Assert.assertFalse(childOp.getDone());
    selOp.process(new VectorizedRowBatch(1), 0);
    childOp.setDone(true);
    // selOp is not done, it will detect child's done=true during the next process(batch) call
    Assert.assertFalse(selOp.getDone());
    Assert.assertTrue(childOp.getDone());
    selOp.process(new VectorizedRowBatch(1), 0);
    // selOp detects child's done=true, so it turns to done=true
    Assert.assertTrue(selOp.getDone());
    Assert.assertTrue(childOp.getDone());
}
Also used : VectorizedRowBatch(org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch) VectorSelectOperator(org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator) CompilationOpContext(org.apache.hadoop.hive.ql.CompilationOpContext) VectorSelectDesc(org.apache.hadoop.hive.ql.plan.VectorSelectDesc) VectorizationContext(org.apache.hadoop.hive.ql.exec.vector.VectorizationContext) SelectDesc(org.apache.hadoop.hive.ql.plan.SelectDesc) VectorSelectDesc(org.apache.hadoop.hive.ql.plan.VectorSelectDesc) Test(org.junit.Test)

Aggregations

CompilationOpContext (org.apache.hadoop.hive.ql.CompilationOpContext)2 VectorSelectOperator (org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator)2 VectorizationContext (org.apache.hadoop.hive.ql.exec.vector.VectorizationContext)2 SelectDesc (org.apache.hadoop.hive.ql.plan.SelectDesc)2 VectorSelectDesc (org.apache.hadoop.hive.ql.plan.VectorSelectDesc)2 ArrayList (java.util.ArrayList)1 HiveConf (org.apache.hadoop.hive.conf.HiveConf)1 VectorizedRowBatch (org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch)1 VectorExpression (org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression)1 ExprNodeColumnDesc (org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc)1 ExprNodeDesc (org.apache.hadoop.hive.ql.plan.ExprNodeDesc)1 ExprNodeGenericFuncDesc (org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc)1 GenericUDF (org.apache.hadoop.hive.ql.udf.generic.GenericUDF)1 GenericUDFOPPlus (org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus)1 BlackholeOperator (org.apache.hive.benchmark.vectorization.BlackholeOperator)1 Test (org.junit.Test)1 Setup (org.openjdk.jmh.annotations.Setup)1