Examples with PRIMITIVE org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE used on opensource projects

Example 81 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class VerifyFastRow method doVerifyDeserializeRead.

public static void doVerifyDeserializeRead(DeserializeRead deserializeRead, TypeInfo typeInfo, Object object, boolean isNull) throws IOException {
    if (isNull) {
        if (object != null) {
            TestCase.fail("Field reports null but object is not null (class " + object.getClass().getName() + ", " + object.toString() + ")");
        }
        return;
    } else if (object == null) {
        TestCase.fail("Field report not null but object is null");
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = deserializeRead.currentBoolean;
                            if (!(object instanceof BooleanWritable)) {
                                TestCase.fail("Boolean expected writable not Boolean");
                            }
                            boolean expected = ((BooleanWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case BYTE:
                        {
                            byte value = deserializeRead.currentByte;
                            if (!(object instanceof ByteWritable)) {
                                TestCase.fail("Byte expected writable not Byte");
                            }
                            byte expected = ((ByteWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                            }
                        }
                        break;
                    case SHORT:
                        {
                            short value = deserializeRead.currentShort;
                            if (!(object instanceof ShortWritable)) {
                                TestCase.fail("Short expected writable not Short");
                            }
                            short expected = ((ShortWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case INT:
                        {
                            int value = deserializeRead.currentInt;
                            if (!(object instanceof IntWritable)) {
                                TestCase.fail("Integer expected writable not Integer");
                            }
                            int expected = ((IntWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case LONG:
                        {
                            long value = deserializeRead.currentLong;
                            if (!(object instanceof LongWritable)) {
                                TestCase.fail("Long expected writable not Long");
                            }
                            Long expected = ((LongWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case FLOAT:
                        {
                            float value = deserializeRead.currentFloat;
                            if (!(object instanceof FloatWritable)) {
                                TestCase.fail("Float expected writable not Float");
                            }
                            float expected = ((FloatWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = deserializeRead.currentDouble;
                            if (!(object instanceof DoubleWritable)) {
                                TestCase.fail("Double expected writable not Double");
                            }
                            double expected = ((DoubleWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case STRING:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            String expected = ((Text) object).toString();
                            if (!string.equals(expected)) {
                                TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
                            }
                        }
                        break;
                    case CHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                            HiveChar expected = ((HiveCharWritable) object).getHiveChar();
                            if (!hiveChar.equals(expected)) {
                                TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
                            }
                        }
                        break;
                    case VARCHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                            HiveVarchar expected = ((HiveVarcharWritable) object).getHiveVarchar();
                            if (!hiveVarchar.equals(expected)) {
                                TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
                            }
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                            if (value == null) {
                                TestCase.fail("Decimal field evaluated to NULL");
                            }
                            HiveDecimal expected = ((HiveDecimalWritable) object).getHiveDecimal();
                            if (!value.equals(expected)) {
                                DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                                int precision = decimalTypeInfo.getPrecision();
                                int scale = decimalTypeInfo.getScale();
                                TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                            }
                        }
                        break;
                    case DATE:
                        {
                            Date value = deserializeRead.currentDateWritable.get();
                            Date expected = ((DateWritableV2) object).get();
                            if (!value.equals(expected)) {
                                TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                            Timestamp expected = ((TimestampWritableV2) object).getTimestamp();
                            if (!value.equals(expected)) {
                                TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                            HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                            HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case BINARY:
                        {
                            byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            BytesWritable bytesWritable = (BytesWritable) object;
                            byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                            if (byteArray.length != expected.length) {
                                TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                            }
                            for (int b = 0; b < byteArray.length; b++) {
                                if (byteArray[b] != expected[b]) {
                                    TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                                }
                            }
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
            throw new Error("Complex types need to be handled separately");
        default:
            throw new Error("Unknown category " + typeInfo.getCategory());
    }
}

Example 82 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class FakeCaptureVectorToRowOutputOperator method process.

@Override
public void process(Object row, int tag) throws HiveException {
    VectorizedRowBatch batch = (VectorizedRowBatch) row;
    boolean selectedInUse = batch.selectedInUse;
    int[] selected = batch.selected;
    for (int logical = 0; logical < batch.size; logical++) {
        int batchIndex = (selectedInUse ? selected[logical] : logical);
        Object[] rowObjects = new Object[outputObjectInspectors.length];
        vectorExtractRow.extractRow(batch, batchIndex, rowObjects);
        for (int c = 0; c < rowObjects.length; c++) {
            switch(outputTypeInfos[c].getCategory()) {
                case PRIMITIVE:
                    rowObjects[c] = ((PrimitiveObjectInspector) outputObjectInspectors[c]).copyObject(rowObjects[c]);
                    break;
                case STRUCT:
                    {
                        final StructTypeInfo structTypeInfo = (StructTypeInfo) outputTypeInfos[c];
                        final StandardStructObjectInspector structInspector = (StandardStructObjectInspector) outputObjectInspectors[c];
                        final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                        final int size = fieldTypeInfos.size();
                        final List<? extends StructField> structFields = structInspector.getAllStructFieldRefs();
                        final Object oldStruct = rowObjects[c];
                        if (oldStruct != null) {
                            List<Object> currentStructData = structInspector.getStructFieldsDataAsList(oldStruct);
                            final Object newStruct = structInspector.create();
                            for (int i = 0; i < size; i++) {
                                final StructField structField = structFields.get(i);
                                final Object oldValue = currentStructData.get(i);
                                final Object newValue;
                                if (oldValue != null) {
                                    newValue = ((PrimitiveObjectInspector) structField.getFieldObjectInspector()).copyObject(oldValue);
                                } else {
                                    newValue = null;
                                }
                                structInspector.setStructFieldData(newStruct, structField, newValue);
                            }
                            rowObjects[c] = ((ArrayList<Object>) newStruct).toArray();
                        }
                    }
                    break;
                default:
                    throw new RuntimeException("Unexpected category " + outputTypeInfos[c].getCategory());
            }
        }
        super.process(rowObjects, 0);
    }
}

Example 83 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class HiveJsonReader method optionallyWrapWritable.

/**
 * The typical usage of this SerDe requires that it return Hadoop Writable
 * objects. However, some uses of this SerDe want the return values to be Java
 * primitive objects. This SerDe works explicitly in Java primitive objects
 * and will wrap the objects in Writable containers if required.
 *
 * @param value The Java primitive object to wrap
 * @param oi The ObjectInspector provides the type to wrap into
 * @return A Hadoop Writable if required; otherwise the object itself
 */
private Object optionallyWrapWritable(final Object value, final ObjectInspector oi) {
    if (!isEnabled(Feature.PRIMITIVE_TO_WRITABLE)) {
        return value;
    }
    final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
    final PrimitiveTypeInfo typeInfo = poi.getTypeInfo();
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(typeInfo).getPrimitiveWritableObject(value);
}

Example 84 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class HiveJsonReader method visitLeafNode.

/**
 * Visit a node if it is expected to be a primitive value (JSON leaf node).
 *
 * @param leafNode The node pointing at the JSON object
 * @param oi The ObjectInspector to parse the value (must be a
 *          PrimitiveObjectInspector)
 * @return A Java primitive Object
 * @throws SerDeException The SerDe is not configured correctly
 */
private Object visitLeafNode(final JsonNode leafNode, final ObjectInspector oi) throws SerDeException {
    final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
    final PrimitiveTypeInfo typeInfo = poi.getTypeInfo();
    if (typeInfo.getPrimitiveCategory() != PrimitiveCategory.STRING) {
        Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.OBJECT);
        Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.ARRAY);
    }
    switch(typeInfo.getPrimitiveCategory()) {
        case INT:
            return Integer.valueOf(leafNode.asInt());
        case BYTE:
            return Byte.valueOf((byte) leafNode.asInt());
        case SHORT:
            return Short.valueOf((short) leafNode.asInt());
        case LONG:
            return Long.valueOf(leafNode.asLong());
        case BOOLEAN:
            return Boolean.valueOf(leafNode.asBoolean());
        case FLOAT:
            return Float.valueOf((float) leafNode.asDouble());
        case DOUBLE:
            return Double.valueOf(leafNode.asDouble());
        case STRING:
            if (leafNode.isValueNode()) {
                return leafNode.asText();
            } else {
                if (isEnabled(Feature.STRINGIFY_COMPLEX_FIELDS)) {
                    return leafNode.toString();
                } else {
                    throw new SerDeException("Complex field found in JSON does not match table definition: " + typeInfo.getTypeName() + ", please consider enabling `" + JsonSerDe.STRINGIFY_COMPLEX + "` table property");
                }
            }
        case BINARY:
            return getByteValue(leafNode);
        case DATE:
            return Date.valueOf(leafNode.asText());
        case TIMESTAMP:
            return tsParser.parseTimestamp(leafNode.asText());
        case DECIMAL:
            return HiveDecimal.create(leafNode.asText());
        case TIMESTAMPLOCALTZ:
            final Timestamp ts = tsParser.parseTimestamp(leafNode.asText());
            final ZoneId zid = ((TimestampLocalTZTypeInfo) typeInfo).timeZone();
            final TimestampTZ tstz = new TimestampTZ();
            tstz.set(ts.toEpochSecond(), ts.getNanos(), zid);
            return tstz;
        case VARCHAR:
            return new HiveVarchar(leafNode.asText(), ((BaseCharTypeInfo) typeInfo).getLength());
        case CHAR:
            return new HiveChar(leafNode.asText(), ((BaseCharTypeInfo) typeInfo).getLength());
        default:
            throw new SerDeException("Could not convert from string to type: " + typeInfo.getTypeName());
    }
}

Example 85 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class HiveJsonReader method visitMapNode.

/**
 * Visit a node if it is expected to be a Map (a.k.a. JSON Object)
 *
 * @param rootNode The node pointing at the JSON object
 * @param oi The ObjectInspector to parse the Map (must be a
 *          MapObjectInspector)
 * @return A Java Map containing the contents of the JSON map
 * @throws SerDeException The SerDe is not configured correctly
 */
private Map<Object, Object> visitMapNode(final JsonNode rootNode, final ObjectInspector oi) throws SerDeException {
    Preconditions.checkArgument(JsonNodeType.OBJECT == rootNode.getNodeType());
    final Map<Object, Object> ret = new LinkedHashMap<>();
    final ObjectInspector mapKeyInspector = ((MapObjectInspector) oi).getMapKeyObjectInspector();
    final ObjectInspector mapValueInspector = ((MapObjectInspector) oi).getMapValueObjectInspector();
    if (!(mapKeyInspector instanceof PrimitiveObjectInspector)) {
        throw new SerDeException("Map key must be a primitive type");
    }
    final Iterator<Entry<String, JsonNode>> it = rootNode.fields();
    while (it.hasNext()) {
        final Entry<String, JsonNode> field = it.next();
        final Object key = visitNode(new TextNode(field.getKey()), mapKeyInspector);
        final Object val = visitNode(field.getValue(), mapValueInspector);
        ret.put(key, val);
    }
    return ret;
}