Examples with AvroMetadata org.apache.beam.sdk.io.AvroSource.AvroMetadata used on opensource projects

Example 1 with AvroMetadata

use of org.apache.beam.sdk.io.AvroSource.AvroMetadata in project beam by apache.

the class AvroSourceTest method testReadSchemaString.

@Test
public void testReadSchemaString() throws Exception {
    List<Bird> expected = createRandomRecords(DEFAULT_RECORD_COUNT);
    String codec = DataFileConstants.NULL_CODEC;
    String filename = generateTestFile(codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec);
    Metadata fileMeta = FileSystems.matchSingleFileSpec(filename);
    AvroMetadata metadata = AvroSource.readMetadataFromFile(fileMeta.resourceId());
    // By default, parse validates the schema, which is what we want.
    Schema schema = new Schema.Parser().parse(metadata.getSchemaString());
    assertEquals(4, schema.getFields().size());
}

Example 2 with AvroMetadata

use of org.apache.beam.sdk.io.AvroSource.AvroMetadata in project beam by apache.

the class AvroSourceTest method testReadMetadataWithCodecs.

@Test
public void testReadMetadataWithCodecs() throws Exception {
    // Test reading files generated using all codecs.
    String[] codecs = { DataFileConstants.NULL_CODEC, DataFileConstants.BZIP2_CODEC, DataFileConstants.DEFLATE_CODEC, DataFileConstants.SNAPPY_CODEC, DataFileConstants.XZ_CODEC };
    List<Bird> expected = createRandomRecords(DEFAULT_RECORD_COUNT);
    for (String codec : codecs) {
        String filename = generateTestFile(codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec);
        Metadata fileMeta = FileSystems.matchSingleFileSpec(filename);
        AvroMetadata metadata = AvroSource.readMetadataFromFile(fileMeta.resourceId());
        assertEquals(codec, metadata.getCodec());
    }
}