Examples with ExprNodeGenericFuncDesc org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc used on opensource projects

Example 36 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project SQLWindowing by hbutani.

the class RuntimeUtils method connectLeadLagFunctionsToPartition.

public static void connectLeadLagFunctionsToPartition(QueryDef qDef, PartitionIterator<Object> pItr) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    List<ExprNodeGenericFuncDesc> llFnDescs = tInfo.getLLInfo().getLeadLagExprs();
    if (llFnDescs == null)
        return;
    for (ExprNodeGenericFuncDesc llFnDesc : llFnDescs) {
        GenericUDFLeadLag llFn = (GenericUDFLeadLag) llFnDesc.getGenericUDF();
        llFn.setpItr(pItr);
    }
}

Example 37 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class VectorizationContext method checkExprNodeDescForDecimal64.

private boolean checkExprNodeDescForDecimal64(ExprNodeDesc exprNodeDesc) throws HiveException {
    if (exprNodeDesc instanceof ExprNodeColumnDesc) {
        int colIndex = getInputColumnIndex((ExprNodeColumnDesc) exprNodeDesc);
        DataTypePhysicalVariation dataTypePhysicalVariation = getDataTypePhysicalVariation(colIndex);
        return (dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64);
    } else if (exprNodeDesc instanceof ExprNodeGenericFuncDesc) {
        // Is the result Decimal64 precision?
        TypeInfo returnType = exprNodeDesc.getTypeInfo();
        if (!checkTypeInfoForDecimal64(returnType)) {
            return false;
        }
        DecimalTypeInfo returnDecimalType = (DecimalTypeInfo) returnType;
        GenericUDF udf = ((ExprNodeGenericFuncDesc) exprNodeDesc).getGenericUDF();
        Class<?> udfClass = udf.getClass();
        // We have a class-level annotation that says whether the UDF's vectorization expressions
        // support Decimal64.
        VectorizedExpressionsSupportDecimal64 annotation = AnnotationUtils.getAnnotation(udfClass, VectorizedExpressionsSupportDecimal64.class);
        if (annotation == null) {
            return false;
        }
        // Carefully check the children to make sure they are Decimal64.
        List<ExprNodeDesc> children = exprNodeDesc.getChildren();
        for (ExprNodeDesc childExprNodeDesc : children) {
            if (childExprNodeDesc instanceof ExprNodeConstantDesc) {
                DecimalTypeInfo childDecimalTypeInfo = decimalTypeFromCastToDecimal(childExprNodeDesc, returnDecimalType);
                if (childDecimalTypeInfo == null) {
                    return false;
                }
                if (!checkTypeInfoForDecimal64(childDecimalTypeInfo)) {
                    return false;
                }
                continue;
            }
            // Otherwise, recurse.
            if (!checkExprNodeDescForDecimal64(childExprNodeDesc)) {
                return false;
            }
        }
        return true;
    } else if (exprNodeDesc instanceof ExprNodeConstantDesc) {
        return checkTypeInfoForDecimal64(exprNodeDesc.getTypeInfo());
    }
    return false;
}

Example 38 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class VectorizationContext method getIdentityExpression.

/**
 * Used as a fast path for operations that don't modify their input, like unary +
 * and casting boolean to long. IdentityExpression and its children are always
 * projections.
 */
private VectorExpression getIdentityExpression(List<ExprNodeDesc> childExprList) throws HiveException {
    ExprNodeDesc childExpr = childExprList.get(0);
    int identityCol;
    TypeInfo identityTypeInfo;
    DataTypePhysicalVariation identityDataTypePhysicalVariation;
    VectorExpression v1 = null;
    if (childExpr instanceof ExprNodeGenericFuncDesc) {
        v1 = getVectorExpression(childExpr);
        identityCol = v1.getOutputColumnNum();
        identityTypeInfo = v1.getOutputTypeInfo();
        identityDataTypePhysicalVariation = v1.getOutputDataTypePhysicalVariation();
    } else if (childExpr instanceof ExprNodeColumnDesc) {
        ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) childExpr;
        identityCol = getInputColumnIndex(colDesc.getColumn());
        identityTypeInfo = colDesc.getTypeInfo();
        // CONSIDER: Validation of type information
        identityDataTypePhysicalVariation = getDataTypePhysicalVariation(identityCol);
    } else {
        throw new HiveException("Expression not supported: " + childExpr);
    }
    VectorExpression ve = new IdentityExpression(identityCol);
    if (v1 != null) {
        ve.setChildExpressions(new VectorExpression[] { v1 });
    }
    ve.setInputTypeInfos(identityTypeInfo);
    ve.setInputDataTypePhysicalVariations(identityDataTypePhysicalVariation);
    ve.setOutputTypeInfo(identityTypeInfo);
    ve.setOutputDataTypePhysicalVariation(identityDataTypePhysicalVariation);
    return ve;
}

Example 39 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class VectorizationContext method getImplicitCastExpression.

/**
 * The GenericUDFs might need their children output to be cast to the given castType.
 * This method returns a cast expression that would achieve the required casting.
 */
private ExprNodeDesc getImplicitCastExpression(GenericUDF udf, ExprNodeDesc child, TypeInfo castType) throws HiveException {
    TypeInfo inputTypeInfo = child.getTypeInfo();
    String inputTypeString = inputTypeInfo.getTypeName();
    String castTypeString = castType.getTypeName();
    if (inputTypeString.equals(castTypeString)) {
        // Nothing to be done
        return null;
    }
    boolean inputTypeDecimal = false;
    boolean castTypeDecimal = false;
    if (decimalTypePattern.matcher(inputTypeString).matches()) {
        inputTypeDecimal = true;
    }
    if (decimalTypePattern.matcher(castTypeString).matches()) {
        castTypeDecimal = true;
    }
    if (castTypeDecimal && !inputTypeDecimal) {
        if (needsImplicitCastForDecimal(udf)) {
            // Cast the input to decimal
            // If castType is decimal, try not to lose precision for numeric types.
            castType = updatePrecision(inputTypeInfo, (DecimalTypeInfo) castType);
            GenericUDFToDecimal castToDecimalUDF = new GenericUDFToDecimal();
            castToDecimalUDF.setTypeInfo(castType);
            List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
            children.add(child);
            ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, castToDecimalUDF, children);
            return desc;
        }
    } else if (!castTypeDecimal && inputTypeDecimal) {
        if (needsImplicitCastForDecimal(udf)) {
            // Cast decimal input to returnType
            GenericUDF genericUdf = getGenericUDFForCast(castType);
            List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
            children.add(child);
            ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, genericUdf, children);
            return desc;
        }
    } else {
        // Casts to exact types including long to double etc. are needed in some special cases.
        if (udf instanceof GenericUDFCoalesce || udf instanceof GenericUDFNvl || udf instanceof GenericUDFElt) {
            GenericUDF genericUdf = getGenericUDFForCast(castType);
            List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
            children.add(child);
            ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, genericUdf, children);
            return desc;
        }
    }
    return null;
}

Example 40 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class VectorizationContext method getStructInExpression.

private VectorExpression getStructInExpression(List<ExprNodeDesc> childExpr, ExprNodeDesc colExpr, TypeInfo colTypeInfo, List<ExprNodeDesc> inChildren, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
    VectorExpression expr = null;
    StructTypeInfo structTypeInfo = (StructTypeInfo) colTypeInfo;
    ArrayList<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
    final int fieldCount = fieldTypeInfos.size();
    ColumnVector.Type[] fieldVectorColumnTypes = new ColumnVector.Type[fieldCount];
    InConstantType[] fieldInConstantTypes = new InConstantType[fieldCount];
    for (int f = 0; f < fieldCount; f++) {
        TypeInfo fieldTypeInfo = fieldTypeInfos.get(f);
        // Only primitive fields supports for now.
        if (fieldTypeInfo.getCategory() != Category.PRIMITIVE) {
            return null;
        }
        // We are going to serialize using the 4 basic types.
        ColumnVector.Type fieldVectorColumnType = getColumnVectorTypeFromTypeInfo(fieldTypeInfo);
        fieldVectorColumnTypes[f] = fieldVectorColumnType;
        // We currently evaluate the IN (..) constants in special ways.
        PrimitiveCategory fieldPrimitiveCategory = ((PrimitiveTypeInfo) fieldTypeInfo).getPrimitiveCategory();
        InConstantType inConstantType = getInConstantTypeFromPrimitiveCategory(fieldPrimitiveCategory);
        fieldInConstantTypes[f] = inConstantType;
    }
    Output buffer = new Output();
    BinarySortableSerializeWrite binarySortableSerializeWrite = new BinarySortableSerializeWrite(fieldCount);
    final int inChildrenCount = inChildren.size();
    byte[][] serializedInChildren = new byte[inChildrenCount][];
    try {
        for (int i = 0; i < inChildrenCount; i++) {
            final ExprNodeDesc node = inChildren.get(i);
            final Object[] constants;
            if (node instanceof ExprNodeConstantDesc) {
                ExprNodeConstantDesc constNode = (ExprNodeConstantDesc) node;
                ConstantObjectInspector output = constNode.getWritableObjectInspector();
                constants = ((List<?>) output.getWritableConstantValue()).toArray();
            } else {
                ExprNodeGenericFuncDesc exprNode = (ExprNodeGenericFuncDesc) node;
                ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprNode);
                ObjectInspector output = evaluator.initialize(exprNode.getWritableObjectInspector());
                constants = (Object[]) evaluator.evaluate(null);
            }
            binarySortableSerializeWrite.set(buffer);
            for (int f = 0; f < fieldCount; f++) {
                Object constant = constants[f];
                if (constant == null) {
                    binarySortableSerializeWrite.writeNull();
                } else {
                    InConstantType inConstantType = fieldInConstantTypes[f];
                    switch(inConstantType) {
                        case STRING_FAMILY:
                            {
                                byte[] bytes;
                                if (constant instanceof Text) {
                                    Text text = (Text) constant;
                                    bytes = text.getBytes();
                                    binarySortableSerializeWrite.writeString(bytes, 0, text.getLength());
                                } else {
                                    throw new HiveException("Unexpected constant String type " + constant.getClass().getSimpleName());
                                }
                            }
                            break;
                        case INT_FAMILY:
                            {
                                long value;
                                if (constant instanceof IntWritable) {
                                    value = ((IntWritable) constant).get();
                                } else if (constant instanceof LongWritable) {
                                    value = ((LongWritable) constant).get();
                                } else {
                                    throw new HiveException("Unexpected constant Long type " + constant.getClass().getSimpleName());
                                }
                                binarySortableSerializeWrite.writeLong(value);
                            }
                            break;
                        case FLOAT_FAMILY:
                            {
                                double value;
                                if (constant instanceof DoubleWritable) {
                                    value = ((DoubleWritable) constant).get();
                                } else {
                                    throw new HiveException("Unexpected constant Double type " + constant.getClass().getSimpleName());
                                }
                                binarySortableSerializeWrite.writeDouble(value);
                            }
                            break;
                        // UNDONE...
                        case DATE:
                        case TIMESTAMP:
                        case DECIMAL:
                        default:
                            throw new RuntimeException("Unexpected IN constant type " + inConstantType.name());
                    }
                }
            }
            serializedInChildren[i] = Arrays.copyOfRange(buffer.getData(), 0, buffer.getLength());
        }
    } catch (Exception e) {
        throw new HiveException(e);
    }
    // Create a single child representing the scratch column where we will
    // generate the serialized keys of the batch.
    int scratchBytesCol = ocm.allocateOutputColumn(TypeInfoFactory.stringTypeInfo);
    Class<?> cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStructColumnInList.class : StructColumnInList.class);
    expr = createVectorExpression(cl, null, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
    ((IStringInExpr) expr).setInListValues(serializedInChildren);
    ((IStructInExpr) expr).setScratchBytesColumn(scratchBytesCol);
    ((IStructInExpr) expr).setStructColumnExprs(this, colExpr.getChildren(), fieldVectorColumnTypes);
    return expr;
}