Examples with VectorExpression org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression used on opensource projects

Example 71 with VectorExpression

use of org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression in project hive by apache.

the class TestVectorizationContext method testIfConditionalExprs.

/**
 * Test that correct VectorExpression classes are chosen for the
 * IF (expr1, expr2, expr3) conditional expression for integer, float,
 * boolean, timestamp and string input types. expr1 is always an input column expression
 * of type long. expr2 and expr3 can be column expressions or constants of other types
 * but must have the same type.
 */
@Test
public void testIfConditionalExprs() throws HiveException {
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Long.class, "col2", "table", false);
    ExprNodeColumnDesc col3Expr = new ExprNodeColumnDesc(Long.class, "col3", "table", false);
    ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc(new Integer(1));
    ExprNodeConstantDesc constDesc3 = new ExprNodeConstantDesc(new Integer(2));
    // long column/column IF
    GenericUDFIf udf = new GenericUDFIf();
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    children1.add(col1Expr);
    children1.add(col2Expr);
    children1.add(col3Expr);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    columns.add("col3");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongColumnLongColumn);
    // long column/scalar IF
    children1.set(2, new ExprNodeConstantDesc(1L));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongColumnLongScalar);
    // long scalar/scalar IF
    children1.set(1, new ExprNodeConstantDesc(1L));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongScalarLongScalar);
    // long scalar/column IF
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongScalarLongColumn);
    // test for double type
    col2Expr = new ExprNodeColumnDesc(Double.class, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(Double.class, "col3", "table", false);
    // double column/column IF
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprDoubleColumnDoubleColumn);
    // double column/scalar IF
    children1.set(2, new ExprNodeConstantDesc(1D));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprDoubleColumnDoubleScalar);
    // double scalar/scalar IF
    children1.set(1, new ExprNodeConstantDesc(1D));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprDoubleScalarDoubleScalar);
    // double scalar/column IF
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprDoubleScalarDoubleColumn);
    // double scalar/long column IF
    children1.set(2, new ExprNodeColumnDesc(Long.class, "col3", "table", false));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprDoubleScalarLongColumn);
    // Additional combinations of (long,double)X(column,scalar) for each of the second
    // and third arguments are omitted. We have coverage of all the source templates
    // already.
    // test for timestamp type
    col2Expr = new ExprNodeColumnDesc(Timestamp.class, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(Timestamp.class, "col3", "table", false);
    // timestamp column/column IF
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprTimestampColumnColumn);
    // timestamp column/scalar IF where scalar is really a CAST of a constant to timestamp.
    ExprNodeGenericFuncDesc f = new ExprNodeGenericFuncDesc();
    f.setGenericUDF(new GenericUDFTimestamp());
    f.setTypeInfo(TypeInfoFactory.timestampTypeInfo);
    List<ExprNodeDesc> children2 = new ArrayList<ExprNodeDesc>();
    f.setChildren(children2);
    children2.add(new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
    children1.set(2, f);
    ve = vc.getVectorExpression(exprDesc);
    // We check for two different classes below because initially the result
    // is IfExprLongColumnLongColumn but in the future if the system is enhanced
    // with constant folding then the result will be IfExprLongColumnLongScalar.
    assertTrue(IfExprTimestampColumnColumn.class == ve.getClass() || IfExprTimestampColumnScalar.class == ve.getClass());
    // timestamp scalar/scalar
    children1.set(1, f);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(IfExprTimestampColumnColumn.class == ve.getClass() || IfExprTimestampScalarScalar.class == ve.getClass());
    // timestamp scalar/column
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(IfExprTimestampColumnColumn.class == ve.getClass() || IfExprTimestampScalarColumn.class == ve.getClass());
    // test for boolean type
    col2Expr = new ExprNodeColumnDesc(Boolean.class, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(Boolean.class, "col3", "table", false);
    // column/column
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongColumnLongColumn);
    // column/scalar IF
    children1.set(2, new ExprNodeConstantDesc(true));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongColumnLongScalar);
    // scalar/scalar IF
    children1.set(1, new ExprNodeConstantDesc(true));
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongScalarLongScalar);
    // scalar/column IF
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprLongScalarLongColumn);
    // test for string type
    constDesc2 = new ExprNodeConstantDesc("Alpha");
    constDesc3 = new ExprNodeConstantDesc("Bravo");
    col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(String.class, "col3", "table", false);
    // column/column
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
    // column/scalar
    children1.set(2, constDesc3);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnStringScalar);
    // scalar/scalar
    children1.set(1, constDesc2);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringScalarStringScalar);
    // scalar/column
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringScalarStringGroupColumn);
    // test for CHAR type
    CharTypeInfo charTypeInfo = new CharTypeInfo(10);
    constDesc2 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Alpha", 10));
    constDesc3 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Bravo", 10));
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(charTypeInfo, "col3", "table", false);
    // column/column
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
    // column/scalar
    children1.set(2, constDesc3);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnCharScalar);
    // scalar/scalar
    children1.set(1, constDesc2);
    // ve = vc.getVectorExpression(exprDesc);
    // assertTrue(ve instanceof IfExprCharScalarCharScalar);
    // scalar/column
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprCharScalarStringGroupColumn);
    // test for VARCHAR type
    VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10);
    constDesc2 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Alpha", 10));
    constDesc3 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Bravo", 10));
    col2Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col2", "table", false);
    col3Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col3", "table", false);
    // column/column
    children1.set(1, col2Expr);
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
    // column/scalar
    children1.set(2, constDesc3);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprStringGroupColumnVarCharScalar);
    // scalar/scalar
    children1.set(1, constDesc2);
    // ve = vc.getVectorExpression(exprDesc);
    // assertTrue(ve instanceof IfExprVarCharScalarVarCharScalar);
    // scalar/column
    children1.set(2, col3Expr);
    ve = vc.getVectorExpression(exprDesc);
    assertTrue(ve instanceof IfExprVarCharScalarStringGroupColumn);
}

Example 72 with VectorExpression

use of org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression in project hive by apache.

the class TestVectorSelectOperator method testSelectOperator.

@Test
public void testSelectOperator() throws HiveException {
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    columns.add("b");
    columns.add("c");
    VectorizationContext vc = new VectorizationContext("name", columns);
    SelectDesc selDesc = new SelectDesc(false);
    List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Long.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Long.class, "b", "table", false);
    ExprNodeColumnDesc colDesc3 = new ExprNodeColumnDesc(Long.class, "c", "table", false);
    ExprNodeGenericFuncDesc plusDesc = new ExprNodeGenericFuncDesc();
    GenericUDF gudf = new GenericUDFOPPlus();
    plusDesc.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    children.add(colDesc1);
    children.add(colDesc2);
    plusDesc.setChildren(children);
    plusDesc.setTypeInfo(TypeInfoFactory.longTypeInfo);
    colList.add(plusDesc);
    colList.add(colDesc3);
    selDesc.setColList(colList);
    List<String> outputColNames = new ArrayList<String>();
    outputColNames.add("_col0");
    outputColNames.add("_col1");
    selDesc.setOutputColumnNames(outputColNames);
    // CONSIDER unwinding ValidatorVectorSelectOperator as a subclass of VectorSelectOperator.
    VectorSelectDesc vectorSelectDesc = new VectorSelectDesc();
    List<ExprNodeDesc> selectColList = selDesc.getColList();
    VectorExpression[] vectorSelectExprs = new VectorExpression[selectColList.size()];
    for (int i = 0; i < selectColList.size(); i++) {
        ExprNodeDesc expr = selectColList.get(i);
        VectorExpression ve = vc.getVectorExpression(expr);
        vectorSelectExprs[i] = ve;
    }
    vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
    vectorSelectDesc.setProjectedOutputColumns(new int[] { 3, 2 });
    ValidatorVectorSelectOperator vso = new ValidatorVectorSelectOperator(new CompilationOpContext(), selDesc, vc, vectorSelectDesc);
    VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(VectorizedRowBatch.DEFAULT_SIZE, 4, 17);
    vso.process(vrg, 0);
}

Example 73 with VectorExpression

use of org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression in project hive by apache.

the class Vectorizer method vectorizeFilterOperator.

public static Operator<? extends OperatorDesc> vectorizeFilterOperator(Operator<? extends OperatorDesc> filterOp, VectorizationContext vContext, VectorFilterDesc vectorFilterDesc) throws HiveException {
    FilterDesc filterDesc = (FilterDesc) filterOp.getConf();
    ExprNodeDesc predicateExpr = filterDesc.getPredicate();
    VectorExpression vectorPredicateExpr = vContext.getVectorExpression(predicateExpr, VectorExpressionDescriptor.Mode.FILTER);
    vectorFilterDesc.setPredicateExpression(vectorPredicateExpr);
    return OperatorFactory.getVectorOperator(filterOp.getCompilationOpContext(), filterDesc, vContext, vectorFilterDesc);
}

Example 74 with VectorExpression

use of org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression in project hive by apache.

the class Vectorizer method canSpecializeReduceSink.

private boolean canSpecializeReduceSink(ReduceSinkDesc desc, boolean isTezOrSpark, VectorizationContext vContext, VectorReduceSinkDesc vectorDesc) throws HiveException {
    VectorReduceSinkInfo vectorReduceSinkInfo = new VectorReduceSinkInfo();
    // Various restrictions.
    // Set this if we encounter a condition we were not expecting.
    boolean isUnexpectedCondition = false;
    boolean isVectorizationReduceSinkNativeEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCESINK_NEW_ENABLED);
    String engine = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
    int limit = desc.getTopN();
    float memUsage = desc.getTopNMemoryUsage();
    boolean hasPTFTopN = (limit >= 0 && memUsage > 0 && desc.isPTFReduceSink());
    boolean hasDistinctColumns = (desc.getDistinctColumnIndices().size() > 0);
    TableDesc keyTableDesc = desc.getKeySerializeInfo();
    Class<? extends Deserializer> keySerializerClass = keyTableDesc.getDeserializerClass();
    boolean isKeyBinarySortable = (keySerializerClass == org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe.class);
    TableDesc valueTableDesc = desc.getValueSerializeInfo();
    Class<? extends Deserializer> valueDeserializerClass = valueTableDesc.getDeserializerClass();
    boolean isValueLazyBinary = (valueDeserializerClass == org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class);
    // We are doing work here we'd normally do in VectorGroupByCommonOperator's constructor.
    // So if we later decide not to specialize, we'll just waste any scratch columns allocated...
    List<ExprNodeDesc> keysDescs = desc.getKeyCols();
    final boolean isEmptyKey = (keysDescs.size() == 0);
    if (!isEmptyKey) {
        VectorExpression[] allKeyExpressions = vContext.getVectorExpressions(keysDescs);
        final int[] reduceSinkKeyColumnMap = new int[allKeyExpressions.length];
        final TypeInfo[] reduceSinkKeyTypeInfos = new TypeInfo[allKeyExpressions.length];
        final Type[] reduceSinkKeyColumnVectorTypes = new Type[allKeyExpressions.length];
        final VectorExpression[] reduceSinkKeyExpressions;
        // Since a key expression can be a calculation and the key will go into a scratch column,
        // we need the mapping and type information.
        ArrayList<VectorExpression> groupByKeyExpressionsList = new ArrayList<VectorExpression>();
        for (int i = 0; i < reduceSinkKeyColumnMap.length; i++) {
            VectorExpression ve = allKeyExpressions[i];
            reduceSinkKeyColumnMap[i] = ve.getOutputColumnNum();
            reduceSinkKeyTypeInfos[i] = keysDescs.get(i).getTypeInfo();
            reduceSinkKeyColumnVectorTypes[i] = VectorizationContext.getColumnVectorTypeFromTypeInfo(reduceSinkKeyTypeInfos[i]);
            if (!IdentityExpression.isColumnOnly(ve)) {
                groupByKeyExpressionsList.add(ve);
            }
        }
        if (groupByKeyExpressionsList.size() == 0) {
            reduceSinkKeyExpressions = null;
        } else {
            reduceSinkKeyExpressions = groupByKeyExpressionsList.toArray(new VectorExpression[0]);
        }
        vectorReduceSinkInfo.setReduceSinkKeyColumnMap(reduceSinkKeyColumnMap);
        vectorReduceSinkInfo.setReduceSinkKeyTypeInfos(reduceSinkKeyTypeInfos);
        vectorReduceSinkInfo.setReduceSinkKeyColumnVectorTypes(reduceSinkKeyColumnVectorTypes);
        vectorReduceSinkInfo.setReduceSinkKeyExpressions(reduceSinkKeyExpressions);
    }
    ArrayList<ExprNodeDesc> valueDescs = desc.getValueCols();
    final boolean isEmptyValue = (valueDescs.size() == 0);
    if (!isEmptyValue) {
        VectorExpression[] allValueExpressions = vContext.getVectorExpressions(valueDescs);
        final int[] reduceSinkValueColumnMap = new int[allValueExpressions.length];
        final TypeInfo[] reduceSinkValueTypeInfos = new TypeInfo[allValueExpressions.length];
        final Type[] reduceSinkValueColumnVectorTypes = new Type[allValueExpressions.length];
        VectorExpression[] reduceSinkValueExpressions;
        ArrayList<VectorExpression> reduceSinkValueExpressionsList = new ArrayList<VectorExpression>();
        for (int i = 0; i < valueDescs.size(); ++i) {
            VectorExpression ve = allValueExpressions[i];
            reduceSinkValueColumnMap[i] = ve.getOutputColumnNum();
            reduceSinkValueTypeInfos[i] = valueDescs.get(i).getTypeInfo();
            reduceSinkValueColumnVectorTypes[i] = VectorizationContext.getColumnVectorTypeFromTypeInfo(reduceSinkValueTypeInfos[i]);
            if (!IdentityExpression.isColumnOnly(ve)) {
                reduceSinkValueExpressionsList.add(ve);
            }
        }
        if (reduceSinkValueExpressionsList.size() == 0) {
            reduceSinkValueExpressions = null;
        } else {
            reduceSinkValueExpressions = reduceSinkValueExpressionsList.toArray(new VectorExpression[0]);
        }
        vectorReduceSinkInfo.setReduceSinkValueColumnMap(reduceSinkValueColumnMap);
        vectorReduceSinkInfo.setReduceSinkValueTypeInfos(reduceSinkValueTypeInfos);
        vectorReduceSinkInfo.setReduceSinkValueColumnVectorTypes(reduceSinkValueColumnVectorTypes);
        vectorReduceSinkInfo.setReduceSinkValueExpressions(reduceSinkValueExpressions);
    }
    boolean useUniformHash = desc.getReducerTraits().contains(UNIFORM);
    vectorReduceSinkInfo.setUseUniformHash(useUniformHash);
    List<ExprNodeDesc> bucketDescs = desc.getBucketCols();
    final boolean isEmptyBuckets = (bucketDescs == null || bucketDescs.size() == 0);
    List<ExprNodeDesc> partitionDescs = desc.getPartitionCols();
    final boolean isEmptyPartitions = (partitionDescs == null || partitionDescs.size() == 0);
    if (useUniformHash || (isEmptyKey && isEmptyBuckets && isEmptyPartitions)) {
    // NOTE: For Uniform Hash or no buckets/partitions, when the key is empty, we will use the VectorReduceSinkEmptyKeyOperator instead.
    } else {
        // Collect bucket and/or partition information for object hashing.
        int[] reduceSinkBucketColumnMap = null;
        TypeInfo[] reduceSinkBucketTypeInfos = null;
        Type[] reduceSinkBucketColumnVectorTypes = null;
        VectorExpression[] reduceSinkBucketExpressions = null;
        if (!isEmptyBuckets) {
            VectorExpression[] allBucketExpressions = vContext.getVectorExpressions(bucketDescs);
            reduceSinkBucketColumnMap = new int[bucketDescs.size()];
            reduceSinkBucketTypeInfos = new TypeInfo[bucketDescs.size()];
            reduceSinkBucketColumnVectorTypes = new Type[bucketDescs.size()];
            ArrayList<VectorExpression> reduceSinkBucketExpressionsList = new ArrayList<VectorExpression>();
            for (int i = 0; i < bucketDescs.size(); ++i) {
                VectorExpression ve = allBucketExpressions[i];
                reduceSinkBucketColumnMap[i] = ve.getOutputColumnNum();
                reduceSinkBucketTypeInfos[i] = bucketDescs.get(i).getTypeInfo();
                reduceSinkBucketColumnVectorTypes[i] = VectorizationContext.getColumnVectorTypeFromTypeInfo(reduceSinkBucketTypeInfos[i]);
                if (!IdentityExpression.isColumnOnly(ve)) {
                    reduceSinkBucketExpressionsList.add(ve);
                }
            }
            if (reduceSinkBucketExpressionsList.size() == 0) {
                reduceSinkBucketExpressions = null;
            } else {
                reduceSinkBucketExpressions = reduceSinkBucketExpressionsList.toArray(new VectorExpression[0]);
            }
        }
        int[] reduceSinkPartitionColumnMap = null;
        TypeInfo[] reduceSinkPartitionTypeInfos = null;
        Type[] reduceSinkPartitionColumnVectorTypes = null;
        VectorExpression[] reduceSinkPartitionExpressions = null;
        if (!isEmptyPartitions) {
            VectorExpression[] allPartitionExpressions = vContext.getVectorExpressions(partitionDescs);
            reduceSinkPartitionColumnMap = new int[partitionDescs.size()];
            reduceSinkPartitionTypeInfos = new TypeInfo[partitionDescs.size()];
            reduceSinkPartitionColumnVectorTypes = new Type[partitionDescs.size()];
            ArrayList<VectorExpression> reduceSinkPartitionExpressionsList = new ArrayList<VectorExpression>();
            for (int i = 0; i < partitionDescs.size(); ++i) {
                VectorExpression ve = allPartitionExpressions[i];
                reduceSinkPartitionColumnMap[i] = ve.getOutputColumnNum();
                reduceSinkPartitionTypeInfos[i] = partitionDescs.get(i).getTypeInfo();
                reduceSinkPartitionColumnVectorTypes[i] = VectorizationContext.getColumnVectorTypeFromTypeInfo(reduceSinkPartitionTypeInfos[i]);
                if (!IdentityExpression.isColumnOnly(ve)) {
                    reduceSinkPartitionExpressionsList.add(ve);
                }
            }
            if (reduceSinkPartitionExpressionsList.size() == 0) {
                reduceSinkPartitionExpressions = null;
            } else {
                reduceSinkPartitionExpressions = reduceSinkPartitionExpressionsList.toArray(new VectorExpression[0]);
            }
        }
        vectorReduceSinkInfo.setReduceSinkBucketColumnMap(reduceSinkBucketColumnMap);
        vectorReduceSinkInfo.setReduceSinkBucketTypeInfos(reduceSinkBucketTypeInfos);
        vectorReduceSinkInfo.setReduceSinkBucketColumnVectorTypes(reduceSinkBucketColumnVectorTypes);
        vectorReduceSinkInfo.setReduceSinkBucketExpressions(reduceSinkBucketExpressions);
        vectorReduceSinkInfo.setReduceSinkPartitionColumnMap(reduceSinkPartitionColumnMap);
        vectorReduceSinkInfo.setReduceSinkPartitionTypeInfos(reduceSinkPartitionTypeInfos);
        vectorReduceSinkInfo.setReduceSinkPartitionColumnVectorTypes(reduceSinkPartitionColumnVectorTypes);
        vectorReduceSinkInfo.setReduceSinkPartitionExpressions(reduceSinkPartitionExpressions);
    }
    // Remember the condition variables for EXPLAIN regardless.
    vectorDesc.setVectorReduceSinkInfo(vectorReduceSinkInfo);
    vectorDesc.setIsVectorizationReduceSinkNativeEnabled(isVectorizationReduceSinkNativeEnabled);
    vectorDesc.setEngine(engine);
    vectorDesc.setIsEmptyKey(isEmptyKey);
    vectorDesc.setIsEmptyValue(isEmptyValue);
    vectorDesc.setIsEmptyBuckets(isEmptyBuckets);
    vectorDesc.setIsEmptyPartitions(isEmptyPartitions);
    vectorDesc.setHasPTFTopN(hasPTFTopN);
    vectorDesc.setHasDistinctColumns(hasDistinctColumns);
    vectorDesc.setIsKeyBinarySortable(isKeyBinarySortable);
    vectorDesc.setIsValueLazyBinary(isValueLazyBinary);
    // This indicates we logged an inconsistency (from our point-of-view) and will not make this
    // operator native...
    vectorDesc.setIsUnexpectedCondition(isUnexpectedCondition);
    // Many restrictions.
    if (!isVectorizationReduceSinkNativeEnabled || !isTezOrSpark || hasPTFTopN || hasDistinctColumns || !isKeyBinarySortable || !isValueLazyBinary || isUnexpectedCondition) {
        return false;
    }
    return true;
}

Example 75 with VectorExpression

use of org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression in project hive by apache.

the class Vectorizer method vectorizeSelectOperator.

public static Operator<? extends OperatorDesc> vectorizeSelectOperator(Operator<? extends OperatorDesc> selectOp, VectorizationContext vContext, VectorSelectDesc vectorSelectDesc) throws HiveException {
    SelectDesc selectDesc = (SelectDesc) selectOp.getConf();
    List<ExprNodeDesc> colList = selectDesc.getColList();
    int index = 0;
    final int size = colList.size();
    VectorExpression[] vectorSelectExprs = new VectorExpression[size];
    int[] projectedOutputColumns = new int[size];
    for (int i = 0; i < size; i++) {
        ExprNodeDesc expr = colList.get(i);
        VectorExpression ve = vContext.getVectorExpression(expr);
        projectedOutputColumns[i] = ve.getOutputColumnNum();
        if (ve instanceof IdentityExpression) {
            // Suppress useless evaluation.
            continue;
        }
        vectorSelectExprs[index++] = ve;
    }
    if (index < size) {
        vectorSelectExprs = Arrays.copyOf(vectorSelectExprs, index);
    }
    vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
    vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns);
    return OperatorFactory.getVectorOperator(selectOp.getCompilationOpContext(), selectDesc, vContext, vectorSelectDesc);
}