Search in sources :

Example 36 with VectorizationContext

use of org.apache.hadoop.hive.ql.exec.vector.VectorizationContext in project hive by apache.

the class VectorGroupByOperatorBench method setup.

@Setup
public void setup() {
    try {
        dataType = dataType.replaceAll("_", ",");
        TypeInfo typeInfo = TypeInfoFactory.getPrimitiveTypeInfo(dataType);
        ColumnVector cv = ColumnVectorGenUtil.generateColumnVector(typeInfo, hasNulls, isRepeating, size, rand);
        TypeDescription typeDescription = TypeDescription.fromString(dataType);
        vrb = typeDescription.createRowBatch(size);
        vrb.size = size;
        vrb.cols[0] = cv;
        VectorizationContext ctx = new VectorizationContext("name", ImmutableList.of("A"));
        GroupByDesc desc = buildGroupByDescType(aggregation, evalMode, "A", typeInfo, processMode);
        Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(new CompilationOpContext(), desc);
        VectorGroupByDesc vectorGroupByDesc = new VectorGroupByDesc();
        vectorGroupByDesc.setProcessingMode(ProcessingMode.HASH);
        vgo = (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx, vectorGroupByDesc);
        vgo.initialize(new Configuration(), null);
    } catch (Exception e) {
        // likely unsupported combination of params
        // https://bugs.openjdk.java.net/browse/CODETOOLS-7901296 is not available yet to skip benchmark cleanly
        System.out.println("Skipping.. Exception: " + e.getMessage());
        System.exit(0);
    }
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) CompilationOpContext(org.apache.hadoop.hive.ql.CompilationOpContext) VectorGroupByDesc(org.apache.hadoop.hive.ql.plan.VectorGroupByDesc) TypeDescription(org.apache.orc.TypeDescription) VectorizationContext(org.apache.hadoop.hive.ql.exec.vector.VectorizationContext) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) VectorGroupByDesc(org.apache.hadoop.hive.ql.plan.VectorGroupByDesc) GroupByDesc(org.apache.hadoop.hive.ql.plan.GroupByDesc) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException) RunnerException(org.openjdk.jmh.runner.RunnerException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) ColumnVector(org.apache.hadoop.hive.ql.exec.vector.ColumnVector) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

VectorizationContext (org.apache.hadoop.hive.ql.exec.vector.VectorizationContext)36 VectorExpression (org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression)19 HiveConf (org.apache.hadoop.hive.conf.HiveConf)18 VectorExtractRow (org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow)18 VectorizedRowBatch (org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch)18 VectorUDFAdaptor (org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor)16 VectorizedRowBatchCtx (org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx)12 ArrayList (java.util.ArrayList)8 CompilationOpContext (org.apache.hadoop.hive.ql.CompilationOpContext)8 VectorRandomRowSource (org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource)7 VectorMapJoinDesc (org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc)7 DataTypePhysicalVariation (org.apache.hadoop.hive.common.type.DataTypePhysicalVariation)6 MapJoinTableContainer (org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer)6 MapJoinTableContainerSerDe (org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe)6 VectorMapJoinOperator (org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator)6 HiveException (org.apache.hadoop.hive.ql.metadata.HiveException)6 ExprNodeGenericFuncDesc (org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc)6 PrimitiveTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)6 TypeInfo (org.apache.hadoop.hive.serde2.typeinfo.TypeInfo)6 List (java.util.List)5