Examples with ReflectDatumReader org.apache.avro.reflect.ReflectDatumReader used on opensource projects

Example 6 with ReflectDatumReader

use of org.apache.avro.reflect.ReflectDatumReader in project flink by apache.

the class AvroOutputFormatITCase method postSubmit.

@Override
protected void postSubmit() throws Exception {
    //compare result for specific user type
    File[] output1;
    File file1 = asFile(outputPath1);
    if (file1.isDirectory()) {
        output1 = file1.listFiles();
        // check for avro ext in dir.
        for (File avroOutput : output1) {
            Assert.assertTrue("Expect extension '.avro'", avroOutput.toString().endsWith(".avro"));
        }
    } else {
        output1 = new File[] { file1 };
    }
    List<String> result1 = new ArrayList<String>();
    DatumReader<User> userDatumReader1 = new SpecificDatumReader<User>(User.class);
    for (File avroOutput : output1) {
        DataFileReader<User> dataFileReader1 = new DataFileReader<User>(avroOutput, userDatumReader1);
        while (dataFileReader1.hasNext()) {
            User user = dataFileReader1.next();
            result1.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
        }
    }
    for (String expectedResult : userData.split("\n")) {
        Assert.assertTrue("expected user " + expectedResult + " not found.", result1.contains(expectedResult));
    }
    //compare result for reflect user type
    File[] output2;
    File file2 = asFile(outputPath2);
    if (file2.isDirectory()) {
        output2 = file2.listFiles();
    } else {
        output2 = new File[] { file2 };
    }
    List<String> result2 = new ArrayList<String>();
    DatumReader<ReflectiveUser> userDatumReader2 = new ReflectDatumReader<ReflectiveUser>(ReflectiveUser.class);
    for (File avroOutput : output2) {
        DataFileReader<ReflectiveUser> dataFileReader2 = new DataFileReader<ReflectiveUser>(avroOutput, userDatumReader2);
        while (dataFileReader2.hasNext()) {
            ReflectiveUser user = dataFileReader2.next();
            result2.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
        }
    }
    for (String expectedResult : userData.split("\n")) {
        Assert.assertTrue("expected user " + expectedResult + " not found.", result2.contains(expectedResult));
    }
}

Example 7 with ReflectDatumReader

use of org.apache.avro.reflect.ReflectDatumReader in project hadoop by apache.

the class AvroTestUtil method testReflect.

public static void testReflect(Object value, Type type, String schema) throws Exception {
    // check that schema matches expected
    Schema s = ReflectData.get().getSchema(type);
    assertEquals(Schema.parse(schema), s);
    // check that value is serialized correctly
    ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(s);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    writer.write(value, EncoderFactory.get().directBinaryEncoder(out, null));
    ReflectDatumReader<Object> reader = new ReflectDatumReader<Object>(s);
    Object after = reader.read(null, DecoderFactory.get().binaryDecoder(out.toByteArray(), null));
    assertEquals(value, after);
}

Example 8 with ReflectDatumReader

use of org.apache.avro.reflect.ReflectDatumReader in project beam by apache.

the class AvroSource method createDatumReader.

private DatumReader<T> createDatumReader() {
    Schema readSchema = getReadSchema();
    Schema fileSchema = getFileSchema();
    checkNotNull(readSchema, "No read schema has been initialized for source %s", this);
    checkNotNull(fileSchema, "No file schema has been initialized for source %s", this);
    if (type == GenericRecord.class) {
        return new GenericDatumReader<>(fileSchema, readSchema);
    } else {
        return new ReflectDatumReader<>(fileSchema, readSchema);
    }
}