Search in sources :

Example 1 with VarcharTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.

the class GenericUDTFGetSplits method convertTypeString.

private TypeDesc convertTypeString(String typeString) throws HiveException {
    TypeDesc typeDesc;
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeString);
    Preconditions.checkState(typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE, "Unsupported non-primitive type " + typeString);
    switch(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
        case BOOLEAN:
            typeDesc = new TypeDesc(TypeDesc.Type.BOOLEAN);
            break;
        case BYTE:
            typeDesc = new TypeDesc(TypeDesc.Type.TINYINT);
            break;
        case SHORT:
            typeDesc = new TypeDesc(TypeDesc.Type.SMALLINT);
            break;
        case INT:
            typeDesc = new TypeDesc(TypeDesc.Type.INT);
            break;
        case LONG:
            typeDesc = new TypeDesc(TypeDesc.Type.BIGINT);
            break;
        case FLOAT:
            typeDesc = new TypeDesc(TypeDesc.Type.FLOAT);
            break;
        case DOUBLE:
            typeDesc = new TypeDesc(TypeDesc.Type.DOUBLE);
            break;
        case STRING:
            typeDesc = new TypeDesc(TypeDesc.Type.STRING);
            break;
        case CHAR:
            CharTypeInfo charTypeInfo = (CharTypeInfo) typeInfo;
            typeDesc = new TypeDesc(TypeDesc.Type.CHAR, charTypeInfo.getLength());
            break;
        case VARCHAR:
            VarcharTypeInfo varcharTypeInfo = (VarcharTypeInfo) typeInfo;
            typeDesc = new TypeDesc(TypeDesc.Type.VARCHAR, varcharTypeInfo.getLength());
            break;
        case DATE:
            typeDesc = new TypeDesc(TypeDesc.Type.DATE);
            break;
        case TIMESTAMP:
            typeDesc = new TypeDesc(TypeDesc.Type.TIMESTAMP);
            break;
        case BINARY:
            typeDesc = new TypeDesc(TypeDesc.Type.BINARY);
            break;
        case DECIMAL:
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            typeDesc = new TypeDesc(TypeDesc.Type.DECIMAL, decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale());
            break;
        default:
            throw new HiveException("Unsupported type " + typeString);
    }
    return typeDesc;
}
Also used : DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) TypeDesc(org.apache.hadoop.hive.llap.TypeDesc) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)

Example 2 with VarcharTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.

the class TestVectorizationContext method testBetweenFilters.

@Test
public void testBetweenFilters() throws HiveException {
    // string tests
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha");
    ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo");
    // string BETWEEN
    GenericUDFBetween udf = new GenericUDFBetween();
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(new Boolean(false)));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringColumnBetween);
    // string NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringColumnNotBetween);
    // CHAR tests
    CharTypeInfo charTypeInfo = new CharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false);
    constDesc = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Alpha", 10));
    constDesc2 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Bravo", 10));
    // CHAR BETWEEN
    udf = new GenericUDFBetween();
    children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(new Boolean(false)));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterCharColumnBetween);
    // CHAR NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterCharColumnNotBetween);
    // VARCHAR tests
    VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    constDesc = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Alpha", 10));
    constDesc2 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Bravo", 10));
    // VARCHAR BETWEEN
    udf = new GenericUDFBetween();
    children1 = new ArrayList<ExprNodeDesc>();
    // no NOT keyword
    children1.add(new ExprNodeConstantDesc(new Boolean(false)));
    children1.add(col1Expr);
    children1.add(constDesc);
    children1.add(constDesc2);
    exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterVarCharColumnBetween);
    // VARCHAR NOT BETWEEN
    // has NOT keyword
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterVarCharColumnNotBetween);
    // long BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(false)));
    children1.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc(10));
    children1.set(3, new ExprNodeConstantDesc(20));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterLongColumnBetween);
    // long NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterLongColumnNotBetween);
    // double BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(false)));
    children1.set(1, new ExprNodeColumnDesc(Double.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc(10.0d));
    children1.set(3, new ExprNodeConstantDesc(20.0d));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterDoubleColumnBetween);
    // double NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterDoubleColumnNotBetween);
    // timestamp BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(false)));
    children1.set(1, new ExprNodeColumnDesc(Timestamp.class, "col1", "table", false));
    children1.set(2, new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
    children1.set(3, new ExprNodeConstantDesc("2013-11-06 00:00:00.000"));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterTimestampColumnBetween.class, ve.getClass());
    // timestamp NOT BETWEEN
    children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertEquals(FilterTimestampColumnNotBetween.class, ve.getClass());
}
Also used : FilterCharColumnBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnBetween) FilterDoubleColumnNotBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) ArrayList(java.util.ArrayList) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) FilterLongColumnBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween) VectorUDFUnixTimeStampTimestamp(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFUnixTimeStampTimestamp) VectorUDFYearTimestamp(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFYearTimestamp) GenericUDFTimestamp(org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp) Timestamp(java.sql.Timestamp) FilterCharColumnNotBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnNotBetween) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) GenericUDFBetween(org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) FilterStringColumnBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) BRoundWithNumDigitsDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.BRoundWithNumDigitsDoubleToDouble) FuncRoundDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncRoundDoubleToDouble) FuncBRoundDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncBRoundDoubleToDouble) FuncLogWithBaseDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseDoubleToDouble) FuncLogWithBaseLongToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseLongToDouble) FuncPowerDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.FuncPowerDoubleToDouble) FuncLnDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncLnDoubleToDouble) FuncSinDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble) RoundWithNumDigitsDoubleToDouble(org.apache.hadoop.hive.ql.exec.vector.expressions.RoundWithNumDigitsDoubleToDouble) FilterDoubleColumnBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween) FilterVarCharColumnNotBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnNotBetween) FilterStringColumnNotBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween) VectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression) DynamicValueVectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.DynamicValueVectorExpression) FilterVarCharColumnBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnBetween) FilterLongColumnNotBetween(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween) Test(org.junit.Test)

Example 3 with VarcharTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.

the class TestVectorizationContext method testFilterStringColCompareStringColumnExpressions.

@Test
public void testFilterStringColCompareStringColumnExpressions() throws HiveException {
    // Strings test
    ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false);
    GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
    ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    columns.add("col2");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // 2 CHAR test
    CharTypeInfo charTypeInfo = new CharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // 2 VARCHAR test
    VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    // Some mix tests (STRING, CHAR), (VARCHAR, CHAR), (VARCHAR, STRING)...
    col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
    col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false);
    col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false);
    udf = new GenericUDFOPGreaterThan();
    exprDesc = new ExprNodeGenericFuncDesc();
    exprDesc.setGenericUDF(udf);
    children1 = new ArrayList<ExprNodeDesc>(2);
    children1.add(col1Expr);
    children1.add(col2Expr);
    exprDesc.setChildren(children1);
    vc = new VectorizationContext("name", columns);
    ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
    assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn);
}
Also used : GenericUDFOPGreaterThan(org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) ArrayList(java.util.ArrayList) FilterStringGroupColGreaterStringGroupColumn(org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringGroupColGreaterStringGroupColumn) ExprNodeGenericFuncDesc(org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) VectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression) DynamicValueVectorExpression(org.apache.hadoop.hive.ql.exec.vector.expressions.DynamicValueVectorExpression) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) Test(org.junit.Test)

Example 4 with VarcharTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.

the class TestVectorSerDeRow method deserializeAndVerify.

void deserializeAndVerify(Output output, DeserializeRead deserializeRead, VectorRandomRowSource source, Object[] expectedRow) throws HiveException, IOException {
    deserializeRead.set(output.getData(), 0, output.getLength());
    PrimitiveCategory[] primitiveCategories = source.primitiveCategories();
    for (int i = 0; i < primitiveCategories.length; i++) {
        Object expected = expectedRow[i];
        PrimitiveCategory primitiveCategory = primitiveCategories[i];
        PrimitiveTypeInfo primitiveTypeInfo = source.primitiveTypeInfos()[i];
        if (!deserializeRead.readNextField()) {
            throw new HiveException("Unexpected NULL when reading primitiveCategory " + primitiveCategory + " expected (" + expected.getClass().getName() + ", " + expected.toString() + ") " + " deserializeRead " + deserializeRead.getClass().getName());
        }
        switch(primitiveCategory) {
            case BOOLEAN:
                {
                    Boolean value = deserializeRead.currentBoolean;
                    BooleanWritable expectedWritable = (BooleanWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    Byte value = deserializeRead.currentByte;
                    ByteWritable expectedWritable = (ByteWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    Short value = deserializeRead.currentShort;
                    ShortWritable expectedWritable = (ShortWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    Integer value = deserializeRead.currentInt;
                    IntWritable expectedWritable = (IntWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    Long value = deserializeRead.currentLong;
                    LongWritable expectedWritable = (LongWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DATE:
                {
                    DateWritable value = deserializeRead.currentDateWritable;
                    DateWritable expectedWritable = (DateWritable) expected;
                    if (!value.equals(expectedWritable)) {
                        TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    Float value = deserializeRead.currentFloat;
                    FloatWritable expectedWritable = (FloatWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    Double value = deserializeRead.currentDouble;
                    DoubleWritable expectedWritable = (DoubleWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
            case CHAR:
            case VARCHAR:
            case BINARY:
                {
                    byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                    Text text = new Text(stringBytes);
                    String string = text.toString();
                    switch(primitiveCategory) {
                        case STRING:
                            {
                                Text expectedWritable = (Text) expected;
                                if (!string.equals(expectedWritable.toString())) {
                                    TestCase.fail("String field mismatch (expected '" + expectedWritable.toString() + "' found '" + string + "')");
                                }
                            }
                            break;
                        case CHAR:
                            {
                                HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                                HiveCharWritable expectedWritable = (HiveCharWritable) expected;
                                if (!hiveChar.equals(expectedWritable.getHiveChar())) {
                                    TestCase.fail("Char field mismatch (expected '" + expectedWritable.getHiveChar() + "' found '" + hiveChar + "')");
                                }
                            }
                            break;
                        case VARCHAR:
                            {
                                HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                                HiveVarcharWritable expectedWritable = (HiveVarcharWritable) expected;
                                if (!hiveVarchar.equals(expectedWritable.getHiveVarchar())) {
                                    TestCase.fail("Varchar field mismatch (expected '" + expectedWritable.getHiveVarchar() + "' found '" + hiveVarchar + "')");
                                }
                            }
                            break;
                        case BINARY:
                            {
                                BytesWritable expectedWritable = (BytesWritable) expected;
                                if (stringBytes.length != expectedWritable.getLength()) {
                                    TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + stringBytes + ")");
                                }
                                byte[] expectedBytes = expectedWritable.getBytes();
                                for (int b = 0; b < stringBytes.length; b++) {
                                    if (stringBytes[b] != expectedBytes[b]) {
                                        TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + stringBytes + ")");
                                    }
                                }
                            }
                            break;
                        default:
                            throw new HiveException("Unexpected primitive category " + primitiveCategory);
                    }
                }
                break;
            case DECIMAL:
                {
                    HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                    if (value == null) {
                        TestCase.fail("Decimal field evaluated to NULL");
                    }
                    HiveDecimalWritable expectedWritable = (HiveDecimalWritable) expected;
                    if (!value.equals(expectedWritable.getHiveDecimal())) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        TestCase.fail("Decimal field mismatch (expected " + expectedWritable.getHiveDecimal() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case TIMESTAMP:
                {
                    Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                    TimestampWritable expectedWritable = (TimestampWritable) expected;
                    if (!value.equals(expectedWritable.getTimestamp())) {
                        TestCase.fail("Timestamp field mismatch (expected " + expectedWritable.getTimestamp() + " found " + value.toString() + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                    HiveIntervalYearMonthWritable expectedWritable = (HiveIntervalYearMonthWritable) expected;
                    HiveIntervalYearMonth expectedValue = expectedWritable.getHiveIntervalYearMonth();
                    if (!value.equals(expectedValue)) {
                        TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                    HiveIntervalDayTimeWritable expectedWritable = (HiveIntervalDayTimeWritable) expected;
                    HiveIntervalDayTime expectedValue = expectedWritable.getHiveIntervalDayTime();
                    if (!value.equals(expectedValue)) {
                        TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
                    }
                }
                break;
            default:
                throw new HiveException("Unexpected primitive category " + primitiveCategory);
        }
    }
    TestCase.assertTrue(deserializeRead.isEndOfInputReached());
}
Also used : HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) DoubleWritable(org.apache.hadoop.hive.serde2.io.DoubleWritable) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) Timestamp(java.sql.Timestamp) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) LongWritable(org.apache.hadoop.io.LongWritable) PrimitiveCategory(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) FloatWritable(org.apache.hadoop.io.FloatWritable) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) BooleanWritable(org.apache.hadoop.io.BooleanWritable)

Example 5 with VarcharTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.

the class VectorRandomRowSource method getWritableObject.

public static Object getWritableObject(int column, Object object, List<ObjectInspector> primitiveObjectInspectorList, PrimitiveCategory[] primitiveCategories, PrimitiveTypeInfo[] primitiveTypeInfos) {
    ObjectInspector objectInspector = primitiveObjectInspectorList.get(column);
    PrimitiveCategory primitiveCategory = primitiveCategories[column];
    PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
    switch(primitiveCategory) {
        case BOOLEAN:
            return ((WritableBooleanObjectInspector) objectInspector).create((boolean) object);
        case BYTE:
            return ((WritableByteObjectInspector) objectInspector).create((byte) object);
        case SHORT:
            return ((WritableShortObjectInspector) objectInspector).create((short) object);
        case INT:
            return ((WritableIntObjectInspector) objectInspector).create((int) object);
        case LONG:
            return ((WritableLongObjectInspector) objectInspector).create((long) object);
        case DATE:
            return ((WritableDateObjectInspector) objectInspector).create((Date) object);
        case FLOAT:
            return ((WritableFloatObjectInspector) objectInspector).create((float) object);
        case DOUBLE:
            return ((WritableDoubleObjectInspector) objectInspector).create((double) object);
        case STRING:
            return ((WritableStringObjectInspector) objectInspector).create((String) object);
        case CHAR:
            {
                WritableHiveCharObjectInspector writableCharObjectInspector = new WritableHiveCharObjectInspector((CharTypeInfo) primitiveTypeInfo);
                return writableCharObjectInspector.create((HiveChar) object);
            }
        case VARCHAR:
            {
                WritableHiveVarcharObjectInspector writableVarcharObjectInspector = new WritableHiveVarcharObjectInspector((VarcharTypeInfo) primitiveTypeInfo);
                return writableVarcharObjectInspector.create((HiveVarchar) object);
            }
        case BINARY:
            return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create((byte[]) object);
        case TIMESTAMP:
            return ((WritableTimestampObjectInspector) objectInspector).create((Timestamp) object);
        case INTERVAL_YEAR_MONTH:
            return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create((HiveIntervalYearMonth) object);
        case INTERVAL_DAY_TIME:
            return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create((HiveIntervalDayTime) object);
        case DECIMAL:
            {
                WritableHiveDecimalObjectInspector writableDecimalObjectInspector = new WritableHiveDecimalObjectInspector((DecimalTypeInfo) primitiveTypeInfo);
                HiveDecimalWritable result = (HiveDecimalWritable) writableDecimalObjectInspector.create((HiveDecimal) object);
                return result;
            }
        default:
            throw new Error("Unknown primitive category " + primitiveCategory);
    }
}
Also used : VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) WritableHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector) WritableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) WritableHiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalDayTimeObjectInspector) WritableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) WritableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) WritableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector) PrimitiveCategory(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory) WritableHiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalYearMonthObjectInspector) WritableHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveCharObjectInspector) WritableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector) WritableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector) WritableHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveCharObjectInspector) WritableHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector) WritableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector) WritableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector) WritableHiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalDayTimeObjectInspector) WritableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector) WritableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) WritableHiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalYearMonthObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) WritableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector) WritableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector) WritableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector) WritableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) WritableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector) WritableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector) WritableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) WritableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector) WritableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector) WritableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector)

Aggregations

VarcharTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo)26 HiveVarchar (org.apache.hadoop.hive.common.type.HiveVarchar)18 CharTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo)18 DecimalTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo)13 HiveChar (org.apache.hadoop.hive.common.type.HiveChar)12 PrimitiveTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)12 Timestamp (java.sql.Timestamp)10 HiveDecimal (org.apache.hadoop.hive.common.type.HiveDecimal)9 Text (org.apache.hadoop.io.Text)9 Test (org.junit.Test)9 HiveVarcharWritable (org.apache.hadoop.hive.serde2.io.HiveVarcharWritable)8 PrimitiveCategory (org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory)8 Date (java.sql.Date)7 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)6 DoubleWritable (org.apache.hadoop.hive.serde2.io.DoubleWritable)5 BytesWritable (org.apache.hadoop.io.BytesWritable)5 LongWritable (org.apache.hadoop.io.LongWritable)5 ArrayList (java.util.ArrayList)4 DeferredJavaObject (org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject)4 DeferredObject (org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject)4