Examples with Deserializer org.apache.hadoop.hive.serde2.Deserializer used on opensource projects

Example 11 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project presto by prestodb.

the class HiveUtil method getTableObjectInspector.

public static StructObjectInspector getTableObjectInspector(@SuppressWarnings("deprecation") Deserializer deserializer) {
    try {
        ObjectInspector inspector = deserializer.getObjectInspector();
        checkArgument(inspector.getCategory() == Category.STRUCT, "expected STRUCT: %s", inspector.getCategory());
        return (StructObjectInspector) inspector;
    } catch (SerDeException e) {
        throw Throwables.propagate(e);
    }
}

Example 12 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project presto by prestodb.

the class HiveUtil method getDeserializer.

@SuppressWarnings("deprecation")
public static Deserializer getDeserializer(Properties schema) {
    String name = getDeserializerClassName(schema);
    Deserializer deserializer = createDeserializer(getDeserializerClass(name));
    initializeDeserializer(deserializer, schema);
    return deserializer;
}

Example 13 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project presto by prestodb.

the class RcFileTester method assertFileContentsOld.

private static <K extends LongWritable, V extends BytesRefArrayWritable> void assertFileContentsOld(Type type, TempFile tempFile, Format format, Iterable<?> expectedValues) throws Exception {
    JobConf configuration = new JobConf(new Configuration(false));
    configuration.set(READ_COLUMN_IDS_CONF_STR, "0");
    configuration.setBoolean(READ_ALL_COLUMNS, false);
    Properties schema = new Properties();
    schema.setProperty(META_TABLE_COLUMNS, "test");
    schema.setProperty(META_TABLE_COLUMN_TYPES, getJavaObjectInspector(type).getTypeName());
    @SuppressWarnings("deprecation") Deserializer deserializer;
    if (format == Format.BINARY) {
        deserializer = new LazyBinaryColumnarSerDe();
    } else {
        deserializer = new ColumnarSerDe();
    }
    deserializer.initialize(configuration, schema);
    configuration.set(SERIALIZATION_LIB, deserializer.getClass().getName());
    InputFormat<K, V> inputFormat = new RCFileInputFormat<>();
    RecordReader<K, V> recordReader = inputFormat.getRecordReader(new FileSplit(new Path(tempFile.getFile().getAbsolutePath()), 0, tempFile.getFile().length(), (String[]) null), configuration, NULL);
    K key = recordReader.createKey();
    V value = recordReader.createValue();
    StructObjectInspector rowInspector = (StructObjectInspector) deserializer.getObjectInspector();
    StructField field = rowInspector.getStructFieldRef("test");
    Iterator<?> iterator = expectedValues.iterator();
    while (recordReader.next(key, value)) {
        Object expectedValue = iterator.next();
        Object rowData = deserializer.deserialize(value);
        Object actualValue = rowInspector.getStructFieldData(rowData, field);
        actualValue = decodeRecordReaderValue(type, actualValue);
        assertColumnValueEquals(type, actualValue, expectedValue);
    }
    assertFalse(iterator.hasNext());
}

Example 14 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project metacat by Netflix.

the class HiveTableUtil method getTableStructFields.

/**
     * getTableStructFields.
     *
     * @param table table
     * @return all struct field refs
     */
public static List<? extends StructField> getTableStructFields(final Table table) {
    final Properties schema = MetaStoreUtils.getTableMetadata(table);
    final String name = schema.getProperty(serdeConstants.SERIALIZATION_LIB);
    if (name == null) {
        return Collections.emptyList();
    }
    final Deserializer deserializer = createDeserializer(getDeserializerClass(name));
    try {
        deserializer.initialize(new Configuration(false), schema);
    } catch (SerDeException e) {
        throw new RuntimeException("error initializing deserializer: " + deserializer.getClass().getName());
    }
    try {
        final ObjectInspector inspector = deserializer.getObjectInspector();
        Preconditions.checkArgument(inspector.getCategory() == ObjectInspector.Category.STRUCT, "expected STRUCT: %s", inspector.getCategory());
        return ((StructObjectInspector) inspector).getAllStructFieldRefs();
    } catch (SerDeException e) {
        throw Throwables.propagate(e);
    }
}

Example 15 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project hive by apache.

the class AvroLazyObjectInspector method deserializeStruct.

/**
   * Deserialize the given struct object
   *
   * @param struct the object to deserialize
   * @param fieldName name of the field on which we are currently operating on
   * @return a deserialized object can hive can further operate on
   * @throws AvroObjectInspectorException if something goes wrong during deserialization
   * */
private Object deserializeStruct(Object struct, String fieldName) {
    byte[] data = ((LazyStruct) struct).getBytes();
    AvroDeserializer deserializer = new AvroDeserializer();
    if (data == null || data.length == 0) {
        return null;
    }
    if (readerSchema == null && schemaRetriever == null) {
        throw new IllegalArgumentException("reader schema or schemaRetriever must be set for field [" + fieldName + "]");
    }
    Schema ws = null;
    Schema rs = null;
    int offset = 0;
    AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable();
    if (readerSchema == null) {
        offset = schemaRetriever.getOffset();
        if (data.length < offset) {
            throw new IllegalArgumentException("Data size cannot be less than [" + offset + "]. Found [" + data.length + "]");
        }
        rs = schemaRetriever.retrieveReaderSchema(data);
        if (rs == null) {
            // still nothing, Raise exception
            throw new IllegalStateException("A valid reader schema could not be retrieved either directly or from the schema retriever for field [" + fieldName + "]");
        }
        ws = schemaRetriever.retrieveWriterSchema(data);
        if (ws == null) {
            throw new IllegalStateException("Null writer schema retrieved from schemaRetriever for field [" + fieldName + "]");
        }
        // adjust the data bytes according to any possible offset that was provided
        if (LOG.isDebugEnabled()) {
            LOG.debug("Retrieved writer Schema: " + ws.toString());
            LOG.debug("Retrieved reader Schema: " + rs.toString());
        }
        try {
            avroWritable.readFields(data, offset, data.length, ws, rs);
        } catch (IOException ioe) {
            throw new AvroObjectInspectorException("Error deserializing avro payload", ioe);
        }
    } else {
        // a reader schema was provided
        if (schemaRetriever != null) {
            // a schema retriever has been provided as well. Attempt to read the write schema from the
            // retriever
            ws = schemaRetriever.retrieveWriterSchema(data);
            if (ws == null) {
                throw new IllegalStateException("Null writer schema retrieved from schemaRetriever for field [" + fieldName + "]");
            }
        } else {
            // attempt retrieving the schema from the data
            ws = retrieveSchemaFromBytes(data);
        }
        rs = readerSchema;
        try {
            avroWritable.readFields(data, ws, rs);
        } catch (IOException ioe) {
            throw new AvroObjectInspectorException("Error deserializing avro payload", ioe);
        }
    }
    AvroObjectInspectorGenerator oiGenerator = null;
    Object deserializedObject = null;
    try {
        oiGenerator = new AvroObjectInspectorGenerator(rs);
        deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs);
    } catch (SerDeException se) {
        throw new AvroObjectInspectorException("Error deserializing avro payload", se);
    }
    return deserializedObject;
}