Examples with SpecificDatumReader org.apache.avro.specific.SpecificDatumReader used on opensource projects

Example 6 with SpecificDatumReader

use of org.apache.avro.specific.SpecificDatumReader in project gora by apache.

the class SolrStore method getDatumReader.

@SuppressWarnings("rawtypes")
private SpecificDatumReader getDatumReader(Schema fieldSchema) {
    SpecificDatumReader<?> reader = readerMap.get(fieldSchema);
    if (reader == null) {
        // ignore dirty bits
        reader = new SpecificDatumReader(fieldSchema);
        SpecificDatumReader localReader = null;
        if ((localReader = readerMap.putIfAbsent(fieldSchema, reader)) != null) {
            reader = localReader;
        }
    }
    return reader;
}

Example 7 with SpecificDatumReader

use of org.apache.avro.specific.SpecificDatumReader in project eiger by wlloyd.

the class SerDeUtils method deserializeWithSchema.

/**
     * Deserializes a single object as stored along with its Schema by serialize(T). NB: See warnings on serialize(T).
     * @param ob An empty object to deserialize into (must not be null).
     * @param bytes Array to deserialize from
     * @throws IOException
     */
public static <T extends SpecificRecord> T deserializeWithSchema(ByteBuffer bytes, T ob) throws IOException {
    BinaryDecoder dec = DIRECT_DECODERS.createBinaryDecoder(ByteBufferUtil.getArray(bytes), null);
    Schema writer = Schema.parse(dec.readString(new Utf8()).toString());
    SpecificDatumReader<T> reader = new SpecificDatumReader<T>(writer);
    reader.setExpected(ob.getSchema());
    return reader.read(ob, dec);
}

Example 8 with SpecificDatumReader

use of org.apache.avro.specific.SpecificDatumReader in project eiger by wlloyd.

the class SerDeUtils method deserialize.

/**
     * Deserializes a single object based on the given Schema.
     * @param writer writer's schema
     * @param bytes Array to deserialize from
     * @param ob An empty object to deserialize into (must not be null).
     * @throws IOException
     */
public static <T extends SpecificRecord> T deserialize(Schema writer, ByteBuffer bytes, T ob) throws IOException {
    BinaryDecoder dec = DIRECT_DECODERS.createBinaryDecoder(ByteBufferUtil.getArray(bytes), null);
    SpecificDatumReader<T> reader = new SpecificDatumReader<T>(writer);
    reader.setExpected(ob.getSchema());
    return reader.read(ob, dec);
}

Example 9 with SpecificDatumReader

use of org.apache.avro.specific.SpecificDatumReader in project flink by apache.

the class AvroRecordInputFormatTest method testDeserializeToSpecificType.

/**
	 * This test validates proper serialization with specific (generated POJO) types.
	 */
@Test
public void testDeserializeToSpecificType() throws IOException {
    DatumReader<User> datumReader = new SpecificDatumReader<User>(userSchema);
    try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
        User rec = dataFileReader.next();
        // check if record has been read correctly
        assertNotNull(rec);
        assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
        assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());
        // now serialize it with our framework:
        ExecutionConfig ec = new ExecutionConfig();
        TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);
        Assert.assertEquals(AvroTypeInfo.class, te.getClass());
        TypeSerializer<User> tser = te.createSerializer(ec);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
            tser.serialize(rec, outView);
        }
        User newRec;
        try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(new ByteArrayInputStream(out.toByteArray()))) {
            newRec = tser.deserialize(inView);
        }
        // check if it is still the same
        assertNotNull(newRec);
        assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
        assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
    }
}

Example 10 with SpecificDatumReader

use of org.apache.avro.specific.SpecificDatumReader in project flink by apache.

the class AvroOutputFormatITCase method postSubmit.

@Override
protected void postSubmit() throws Exception {
    //compare result for specific user type
    File[] output1;
    File file1 = asFile(outputPath1);
    if (file1.isDirectory()) {
        output1 = file1.listFiles();
        // check for avro ext in dir.
        for (File avroOutput : output1) {
            Assert.assertTrue("Expect extension '.avro'", avroOutput.toString().endsWith(".avro"));
        }
    } else {
        output1 = new File[] { file1 };
    }
    List<String> result1 = new ArrayList<String>();
    DatumReader<User> userDatumReader1 = new SpecificDatumReader<User>(User.class);
    for (File avroOutput : output1) {
        DataFileReader<User> dataFileReader1 = new DataFileReader<User>(avroOutput, userDatumReader1);
        while (dataFileReader1.hasNext()) {
            User user = dataFileReader1.next();
            result1.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
        }
    }
    for (String expectedResult : userData.split("\n")) {
        Assert.assertTrue("expected user " + expectedResult + " not found.", result1.contains(expectedResult));
    }
    //compare result for reflect user type
    File[] output2;
    File file2 = asFile(outputPath2);
    if (file2.isDirectory()) {
        output2 = file2.listFiles();
    } else {
        output2 = new File[] { file2 };
    }
    List<String> result2 = new ArrayList<String>();
    DatumReader<ReflectiveUser> userDatumReader2 = new ReflectDatumReader<ReflectiveUser>(ReflectiveUser.class);
    for (File avroOutput : output2) {
        DataFileReader<ReflectiveUser> dataFileReader2 = new DataFileReader<ReflectiveUser>(avroOutput, userDatumReader2);
        while (dataFileReader2.hasNext()) {
            ReflectiveUser user = dataFileReader2.next();
            result2.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
        }
    }
    for (String expectedResult : userData.split("\n")) {
        Assert.assertTrue("expected user " + expectedResult + " not found.", result2.contains(expectedResult));
    }
}