Examples with Deserializer org.apache.hadoop.hive.serde2.Deserializer used on opensource projects

Example 61 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project presto by prestodb.

the class HiveMetadata method columnMetadataGetter.

@VisibleForTesting
static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(Table table, TypeManager typeManager, ColumnConverter columnConverter) {
    ImmutableList.Builder<String> columnNames = ImmutableList.builder();
    table.getPartitionColumns().stream().map(Column::getName).forEach(columnNames::add);
    table.getDataColumns().stream().map(Column::getName).forEach(columnNames::add);
    List<String> allColumnNames = columnNames.build();
    if (allColumnNames.size() > Sets.newHashSet(allColumnNames).size()) {
        throw new PrestoException(HIVE_INVALID_METADATA, format("Hive metadata for table %s is invalid: Table descriptor contains duplicate columns", table.getTableName()));
    }
    List<Column> tableColumns = table.getDataColumns();
    ImmutableMap.Builder<String, Optional<String>> builder = ImmutableMap.builder();
    ImmutableMap.Builder<String, Optional<String>> typeMetadataBuilder = ImmutableMap.builder();
    for (Column field : concat(tableColumns, table.getPartitionColumns())) {
        if (field.getComment().isPresent() && !field.getComment().get().equals("from deserializer")) {
            builder.put(field.getName(), field.getComment());
        } else {
            builder.put(field.getName(), Optional.empty());
        }
        typeMetadataBuilder.put(field.getName(), field.getTypeMetadata());
    }
    // add hidden columns
    builder.put(PATH_COLUMN_NAME, Optional.empty());
    if (table.getStorage().getBucketProperty().isPresent()) {
        builder.put(BUCKET_COLUMN_NAME, Optional.empty());
    }
    builder.put(FILE_SIZE_COLUMN_NAME, Optional.empty());
    builder.put(FILE_MODIFIED_TIME_COLUMN_NAME, Optional.empty());
    Map<String, Optional<String>> columnComment = builder.build();
    Map<String, Optional<String>> typeMetadata = typeMetadataBuilder.build();
    return handle -> new ColumnMetadata(handle.getName(), typeManager.getType(columnConverter.getTypeSignature(handle.getHiveType(), typeMetadata.getOrDefault(handle.getName(), Optional.empty()))), columnComment.get(handle.getName()).orElse(null), columnExtraInfo(handle.isPartitionKey()), handle.isHidden());
}

Example 62 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project presto by prestodb.

the class HiveUtil method getDeserializer.

public static Deserializer getDeserializer(Configuration configuration, Properties schema) {
    String name = getDeserializerClassName(schema);
    // see also https://issues.apache.org/jira/browse/HIVE-16922
    if (name.equals("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) {
        if (schema.containsKey("colelction.delim") && !schema.containsKey(COLLECTION_DELIM)) {
            schema.put(COLLECTION_DELIM, schema.getProperty("colelction.delim"));
        }
    }
    Deserializer deserializer = createDeserializer(getDeserializerClass(name));
    initializeDeserializer(configuration, deserializer, schema);
    return deserializer;
}

Example 63 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project flink by apache.

the class HiveShimV100 method getFieldsFromDeserializer.

@Override
public List<FieldSchema> getFieldsFromDeserializer(Configuration conf, Table table, boolean skipConfError) {
    try {
        Method utilMethod = getHiveMetaStoreUtilsClass().getMethod("getDeserializer", Configuration.class, Table.class);
        Deserializer deserializer = (Deserializer) utilMethod.invoke(null, conf, table);
        utilMethod = getHiveMetaStoreUtilsClass().getMethod("getFieldsFromDeserializer", String.class, Deserializer.class);
        return (List<FieldSchema>) utilMethod.invoke(null, table.getTableName(), deserializer);
    } catch (Exception e) {
        throw new CatalogException("Failed to get table schema from deserializer", e);
    }
}

Example 64 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project drill by apache.

the class HiveDefaultRecordReader method createDeserializer.

private static Deserializer createDeserializer(JobConf job, StorageDescriptor sd, Properties properties) throws Exception {
    final Class<? extends Deserializer> c = Class.forName(sd.getSerdeInfo().getSerializationLib()).asSubclass(Deserializer.class);
    final Deserializer deserializer = c.getConstructor().newInstance();
    deserializer.initialize(job, properties);
    return deserializer;
}

Example 65 with Deserializer

use of org.apache.hadoop.hive.serde2.Deserializer in project incubator-gobblin by apache.

the class HiveMetaStoreUtils method getDeserializer.

/**
 * Returns a Deserializer from HiveRegistrationUnit if present and successfully initialized. Else returns null.
 */
private static Deserializer getDeserializer(HiveRegistrationUnit unit) {
    Optional<String> serdeClass = unit.getSerDeType();
    if (!serdeClass.isPresent()) {
        return null;
    }
    String serde = serdeClass.get();
    HiveConf hiveConf;
    Deserializer deserializer;
    try {
        hiveConf = SharedResourcesBrokerFactory.getImplicitBroker().getSharedResource(new HiveConfFactory<>(), SharedHiveConfKey.INSTANCE);
        deserializer = ReflectionUtils.newInstance(hiveConf.getClassByName(serde).asSubclass(Deserializer.class), hiveConf);
    } catch (ClassNotFoundException e) {
        LOG.warn("Serde class " + serde + " not found!", e);
        return null;
    } catch (NotConfiguredException nce) {
        LOG.error("Implicit broker is not configured properly", nce);
        return null;
    }
    Properties props = new Properties();
    props.putAll(unit.getProps().getProperties());
    props.putAll(unit.getStorageProps().getProperties());
    props.putAll(unit.getSerDeProps().getProperties());
    try {
        SerDeUtils.initializeSerDe(deserializer, hiveConf, props, null);
        // handling in AvroSerDe added in HIVE-7868.
        if (deserializer instanceof AvroSerDe) {
            try {
                inVokeDetermineSchemaOrThrowExceptionMethod(props, new Configuration());
            } catch (SchemaParseException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) {
                LOG.warn("Failed to initialize AvroSerDe.");
                throw new SerDeException(e);
            }
        }
    } catch (SerDeException e) {
        LOG.warn("Failed to initialize serde " + serde + " with properties " + props + " for table " + unit.getDbName() + "." + unit.getTableName());
        return null;
    }
    return deserializer;
}