Examples with HiveVarchar org.apache.hadoop.hive.common.type.HiveVarchar used on opensource projects

Example 21 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorStringExpressions method testVarCharScalarConcatCol.

@Test
public void testVarCharScalarConcatCol() throws HiveException {
    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatch();
    StringScalarConcatStringGroupCol expr = new StringScalarConcatStringGroupCol(new HiveVarchar(new String(red), 14).getValue().getBytes(), 0, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    int cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);
    int cmp2 = StringExpr.compare(redgreen, 0, redgreen.length, outCol.vector[1], outCol.start[1], outCol.length[1]);
    Assert.assertEquals(0, cmp2);
    Assert.assertFalse(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    // no nulls, not repeating
    batch = makeStringBatch();
    batch.cols[0].noNulls = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    cmp2 = StringExpr.compare(redgreen, 0, redgreen.length, outCol.vector[1], outCol.start[1], outCol.length[1]);
    Assert.assertEquals(0, cmp2);
    int cmp3 = StringExpr.compare(red, 0, red.length, outCol.vector[2], outCol.start[2], outCol.length[2]);
    Assert.assertEquals(0, cmp3);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    // has nulls, is repeating
    batch = makeStringBatch();
    batch.cols[0].isRepeating = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isRepeating);
    // no nulls, is repeating
    batch = makeStringBatch();
    batch.cols[0].isRepeating = true;
    batch.cols[0].noNulls = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isRepeating);
    Assert.assertTrue(outCol.noNulls);
}

Example 22 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class ExprNodeDescExprFactory method interpretConstantAsPrimitive.

/**
 * {@inheritDoc}
 */
@Override
protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue, PrimitiveTypeInfo sourceType, boolean isEqual) {
    if (constantValue instanceof Number || constantValue instanceof String) {
        try {
            PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
            if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantValue.toString()).intValueExact();
            } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantValue.toString()).longValueExact();
            } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
                return Double.valueOf(constantValue.toString());
            } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
                return Float.valueOf(constantValue.toString());
            } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantValue.toString()).byteValueExact();
            } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantValue.toString()).shortValueExact();
            } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
                return HiveDecimal.create(constantValue.toString());
            }
        } catch (NumberFormatException | ArithmeticException nfe) {
            if (!isEqual && (constantValue instanceof Number || NumberUtils.isNumber(constantValue.toString()))) {
                // type conversion for us.
                return constantValue;
            }
            LOG.trace("Failed to narrow type of constant", nfe);
            return null;
        }
    }
    // Comparision of decimal and float/double happens in float/double.
    if (constantValue instanceof HiveDecimal) {
        HiveDecimal hiveDecimal = (HiveDecimal) constantValue;
        PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
        if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
            return hiveDecimal.doubleValue();
        } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
            return hiveDecimal.floatValue();
        }
        return hiveDecimal;
    }
    String constTypeInfoName = sourceType.getTypeName();
    if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
        // appropriate type.
        if (targetType instanceof CharTypeInfo) {
            final String constValue = constantValue.toString();
            final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
            HiveChar newValue = new HiveChar(constValue, length);
            HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
            if (maxCharConst.equals(newValue)) {
                return newValue;
            } else {
                return null;
            }
        }
        if (targetType instanceof VarcharTypeInfo) {
            final String constValue = constantValue.toString();
            final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
            HiveVarchar newValue = new HiveVarchar(constValue, length);
            HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
            if (maxCharConst.equals(newValue)) {
                return newValue;
            } else {
                return null;
            }
        }
    }
    return constantValue;
}

Example 23 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class RexNodeExprFactory method interpretConstantAsPrimitive.

/**
 * {@inheritDoc}
 */
@Override
protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue, PrimitiveTypeInfo sourceType, boolean isEqual) {
    // Extract string value if necessary
    Object constantToInterpret = constantValue;
    if (constantValue instanceof NlsString) {
        constantToInterpret = ((NlsString) constantValue).getValue();
    }
    if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
        try {
            PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
            if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString()).intValueExact();
            } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString()).longValueExact();
            } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString());
            } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString());
            } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString()).byteValueExact();
            } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
                return toBigDecimal(constantToInterpret.toString()).shortValueExact();
            } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
                HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
                return decimal != null ? decimal.bigDecimalValue() : null;
            }
        } catch (NumberFormatException | ArithmeticException nfe) {
            if (!isEqual && (constantToInterpret instanceof Number || NumberUtils.isNumber(constantToInterpret.toString()))) {
                // type conversion for us.
                return constantToInterpret;
            }
            LOG.trace("Failed to narrow type of constant", nfe);
            return null;
        }
    }
    if (constantToInterpret instanceof BigDecimal) {
        return constantToInterpret;
    }
    String constTypeInfoName = sourceType.getTypeName();
    if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
        // appropriate type.
        if (targetType instanceof CharTypeInfo) {
            final String constValue = constantToInterpret.toString();
            final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
            HiveChar newValue = new HiveChar(constValue, length);
            HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
            if (maxCharConst.equals(newValue)) {
                return makeHiveUnicodeString(newValue.getValue());
            } else {
                return null;
            }
        }
        if (targetType instanceof VarcharTypeInfo) {
            final String constValue = constantToInterpret.toString();
            final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
            HiveVarchar newValue = new HiveVarchar(constValue, length);
            HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
            if (maxCharConst.equals(newValue)) {
                return makeHiveUnicodeString(newValue.getValue());
            } else {
                return null;
            }
        }
    }
    return constantValue;
}

Example 24 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorizationContext method testBetweenFilters.

@Test
public void testBetweenFilters() throws HiveException {
    // string tests
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha");
    ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo");
    // string BETWEEN
    GenericUDFBetween udf = new GenericUDFBetween();
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(Boolean.FALSE));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringColumnBetween);
    // string NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringColumnNotBetween);
    // CHAR tests
    CharTypeInfo charTypeInfo = new CharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false);
    constDesc = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Alpha", 10));
    constDesc2 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Bravo", 10));
    // CHAR BETWEEN
    udf = new GenericUDFBetween();
    children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(Boolean.FALSE));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterCharColumnBetween);
    // CHAR NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterCharColumnNotBetween);
    // VARCHAR tests
    VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    constDesc = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Alpha", 10));
    constDesc2 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Bravo", 10));
    // VARCHAR BETWEEN
    udf = new GenericUDFBetween();
    children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(Boolean.FALSE));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterVarCharColumnBetween);
    // VARCHAR NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterVarCharColumnNotBetween);
    // long BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.FALSE));
    children1.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc(10));
    children1.set(3, new ExprNodeConstantDesc(20));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterLongColumnBetween);
    // long NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterLongColumnNotBetween);
    // double BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.FALSE));
    children1.set(1, new ExprNodeColumnDesc(Double.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc(10.0d));
    children1.set(3, new ExprNodeConstantDesc(20.0d));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterDoubleColumnBetween);
    // double NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterDoubleColumnNotBetween);
    // timestamp BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.FALSE));
    children1.set(1, new ExprNodeColumnDesc(Timestamp.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
    children1.set(3, new ExprNodeConstantDesc("2013-11-06 00:00:00.000"));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterTimestampColumnBetween.class, ve.getClass());
    // timestamp NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(Boolean.TRUE));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterTimestampColumnNotBetween.class, ve.getClass());
}

Example 25 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VerifyFastRow method serializeWrite.

public static void serializeWrite(SerializeWrite serializeWrite, TypeInfo typeInfo, Object object) throws IOException {
    if (object == null) {
        serializeWrite.writeNull();
        return;
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = ((BooleanWritable) object).get();
                            serializeWrite.writeBoolean(value);
                        }
                        break;
                    case BYTE:
                        {
                            byte value = ((ByteWritable) object).get();
                            serializeWrite.writeByte(value);
                        }
                        break;
                    case SHORT:
                        {
                            short value = ((ShortWritable) object).get();
                            serializeWrite.writeShort(value);
                        }
                        break;
                    case INT:
                        {
                            int value = ((IntWritable) object).get();
                            serializeWrite.writeInt(value);
                        }
                        break;
                    case LONG:
                        {
                            long value = ((LongWritable) object).get();
                            serializeWrite.writeLong(value);
                        }
                        break;
                    case FLOAT:
                        {
                            float value = ((FloatWritable) object).get();
                            serializeWrite.writeFloat(value);
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = ((DoubleWritable) object).get();
                            serializeWrite.writeDouble(value);
                        }
                        break;
                    case STRING:
                        {
                            Text value = (Text) object;
                            byte[] stringBytes = value.getBytes();
                            int stringLength = stringBytes.length;
                            serializeWrite.writeString(stringBytes, 0, stringLength);
                        }
                        break;
                    case CHAR:
                        {
                            HiveChar value = ((HiveCharWritable) object).getHiveChar();
                            serializeWrite.writeHiveChar(value);
                        }
                        break;
                    case VARCHAR:
                        {
                            HiveVarchar value = ((HiveVarcharWritable) object).getHiveVarchar();
                            serializeWrite.writeHiveVarchar(value);
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = ((HiveDecimalWritable) object).getHiveDecimal();
                            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                            serializeWrite.writeHiveDecimal(value, decTypeInfo.scale());
                        }
                        break;
                    case DATE:
                        {
                            Date value = ((DateWritableV2) object).get();
                            serializeWrite.writeDate(value);
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = ((TimestampWritableV2) object).getTimestamp();
                            serializeWrite.writeTimestamp(value);
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            serializeWrite.writeHiveIntervalYearMonth(value);
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            serializeWrite.writeHiveIntervalDayTime(value);
                        }
                        break;
                    case BINARY:
                        {
                            BytesWritable byteWritable = (BytesWritable) object;
                            byte[] binaryBytes = byteWritable.getBytes();
                            int length = byteWritable.getLength();
                            serializeWrite.writeBinary(binaryBytes, 0, length);
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory().name());
                }
            }
            break;
        case LIST:
            {
                ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
                TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
                ArrayList<Object> elements = (ArrayList<Object>) object;
                serializeWrite.beginList(elements);
                boolean isFirst = true;
                for (Object elementObject : elements) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateList();
                    }
                    if (elementObject == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, elementTypeInfo, elementObject);
                    }
                }
                serializeWrite.finishList();
            }
            break;
        case MAP:
            {
                MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
                TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
                TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
                Map<Object, Object> hashMap = (Map<Object, Object>) object;
                serializeWrite.beginMap(hashMap);
                boolean isFirst = true;
                for (Map.Entry<Object, Object> entry : hashMap.entrySet()) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateKeyValuePair();
                    }
                    if (entry.getKey() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, keyTypeInfo, entry.getKey());
                    }
                    serializeWrite.separateKey();
                    if (entry.getValue() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, valueTypeInfo, entry.getValue());
                    }
                }
                serializeWrite.finishMap();
            }
            break;
        case STRUCT:
            {
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                List<Object> fieldValues = (List<Object>) object;
                final int size = fieldValues.size();
                serializeWrite.beginStruct(fieldValues);
                boolean isFirst = true;
                for (int i = 0; i < size; i++) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateStruct();
                    }
                    serializeWrite(serializeWrite, fieldTypeInfos.get(i), fieldValues.get(i));
                }
                serializeWrite.finishStruct();
            }
            break;
        case UNION:
            {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
                final int size = fieldTypeInfos.size();
                StandardUnionObjectInspector.StandardUnion standardUnion = (StandardUnionObjectInspector.StandardUnion) object;
                byte tag = standardUnion.getTag();
                serializeWrite.beginUnion(tag);
                serializeWrite(serializeWrite, fieldTypeInfos.get(tag), standardUnion.getObject());
                serializeWrite.finishUnion();
            }
            break;
        default:
            throw new Error("Unknown category " + typeInfo.getCategory().name());
    }
}