Examples with AvroSerDe org.apache.hadoop.hive.serde2.avro.AvroSerDe used on opensource projects

Example 1 with AvroSerDe

use of org.apache.hadoop.hive.serde2.avro.AvroSerDe in project hive by apache.

the class TestAvroSerde method initializeDoesNotReuseSchemasFromConf.

@Test
public void initializeDoesNotReuseSchemasFromConf() throws SerDeException {
    // Hive will re-use the Configuration object that it passes in to be
    // initialized.  Therefore we need to make sure we don't look for any
    // old schemas within it.
    Configuration conf = new Configuration();
    conf.set(AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), originalSchema.toString(false));
    Properties props = new Properties();
    props.put(AvroTableProperties.SCHEMA_LITERAL.getPropName(), newSchemaString);
    AvroSerDe asd = new AvroSerDe();
    SerDeUtils.initializeSerDe(asd, conf, props, null);
    // Verify that the schema now within the configuration is the one passed
    // in via the properties
    assertEquals(newSchema, AvroSerdeUtils.getSchemaFor(conf.get(AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName())));
}

Example 2 with AvroSerDe

use of org.apache.hadoop.hive.serde2.avro.AvroSerDe in project haivvreo by jghoman.

the class TestAvroSerde method verifyErrorSchemaReturned.

private void verifyErrorSchemaReturned(Properties props) throws SerDeException {
    AvroSerDe asd = new AvroSerDe();
    asd.initialize(new Configuration(), props);
    assertTrue(asd.getObjectInspector() instanceof StandardStructObjectInspector);
    StandardStructObjectInspector oi = (StandardStructObjectInspector) asd.getObjectInspector();
    List<? extends StructField> allStructFieldRefs = oi.getAllStructFieldRefs();
    assertEquals(SchemaResolutionProblem.SIGNAL_BAD_SCHEMA.getFields().size(), allStructFieldRefs.size());
    StructField firstField = allStructFieldRefs.get(0);
    assertTrue(firstField.toString().contains("error_error_error_error_error_error_error"));
    try {
        Writable mock = Mockito.mock(Writable.class);
        asd.deserialize(mock);
        fail("Should have thrown a BadSchemaException");
    } catch (BadSchemaException bse) {
    // good
    }
    try {
        Object o = Mockito.mock(Object.class);
        ObjectInspector mockOI = Mockito.mock(ObjectInspector.class);
        asd.serialize(o, mockOI);
        fail("Should have thrown a BadSchemaException");
    } catch (BadSchemaException bse) {
    // good
    }
}

Example 3 with AvroSerDe

use of org.apache.hadoop.hive.serde2.avro.AvroSerDe in project hive by apache.

the class TestAvroSerde method verifyExpectedException.

private void verifyExpectedException(Properties props) {
    AvroSerDe asd = new AvroSerDe();
    try {
        SerDeUtils.initializeSerDe(asd, new Configuration(), props, null);
        fail("Expected Exception did not be thrown");
    } catch (SerDeException e) {
    // good
    }
}

Example 4 with AvroSerDe

use of org.apache.hadoop.hive.serde2.avro.AvroSerDe in project incubator-gobblin by apache.

the class HiveMetaStoreUtils method getDeserializer.

/**
 * Returns a Deserializer from HiveRegistrationUnit if present and successfully initialized. Else returns null.
 */
private static Deserializer getDeserializer(HiveRegistrationUnit unit) {
    Optional<String> serdeClass = unit.getSerDeType();
    if (!serdeClass.isPresent()) {
        return null;
    }
    String serde = serdeClass.get();
    HiveConf hiveConf = new HiveConf();
    Deserializer deserializer;
    try {
        deserializer = ReflectionUtils.newInstance(hiveConf.getClassByName(serde).asSubclass(Deserializer.class), hiveConf);
    } catch (ClassNotFoundException e) {
        LOG.warn("Serde class " + serde + " not found!", e);
        return null;
    }
    Properties props = new Properties();
    props.putAll(unit.getProps().getProperties());
    props.putAll(unit.getStorageProps().getProperties());
    props.putAll(unit.getSerDeProps().getProperties());
    try {
        SerDeUtils.initializeSerDe(deserializer, hiveConf, props, null);
        // handling in AvroSerDe added in HIVE-7868.
        if (deserializer instanceof AvroSerDe) {
            try {
                inVokeDetermineSchemaOrThrowExceptionMethod(props, new Configuration());
            } catch (SchemaParseException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) {
                LOG.warn("Failed to initialize AvroSerDe.");
                throw new SerDeException(e);
            }
        }
    } catch (SerDeException e) {
        LOG.warn("Failed to initialize serde " + serde + " with properties " + props + " for table " + unit.getDbName() + "." + unit.getTableName());
        return null;
    }
    return deserializer;
}