Examples with GenericDatumReader org.apache.avro.generic.GenericDatumReader used on opensource projects

Example 41 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project voldemort by voldemort.

the class ClientConfigUtil method readMultipleClientConfigAvro.

/**
     * Parses a string that contains multiple fat client configs in avro format
     * 
     * @param configAvro Input string of avro format, that contains config for
     *        multiple stores
     * @return Map of store names to store config properties
     */
@SuppressWarnings("unchecked")
public static Map<String, Properties> readMultipleClientConfigAvro(String configAvro) {
    Map<String, Properties> mapStoreToProps = Maps.newHashMap();
    try {
        JsonDecoder decoder = new JsonDecoder(CLIENT_CONFIGS_AVRO_SCHEMA, configAvro);
        GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(CLIENT_CONFIGS_AVRO_SCHEMA);
        Map<Utf8, Map<Utf8, Utf8>> storeConfigs = (Map<Utf8, Map<Utf8, Utf8>>) datumReader.read(null, decoder);
        // Store config props to return back
        for (Utf8 storeName : storeConfigs.keySet()) {
            Properties props = new Properties();
            Map<Utf8, Utf8> singleConfig = storeConfigs.get(storeName);
            for (Utf8 key : singleConfig.keySet()) {
                props.put(key.toString(), singleConfig.get(key).toString());
            }
            if (storeName == null || storeName.length() == 0) {
                throw new Exception("Invalid store name found!");
            }
            mapStoreToProps.put(storeName.toString(), props);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return mapStoreToProps;
}

Example 42 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project beam by apache.

the class AvroSource method createDatumReader.

private DatumReader<T> createDatumReader() {
    Schema readSchema = getReadSchema();
    Schema fileSchema = getFileSchema();
    checkNotNull(readSchema, "No read schema has been initialized for source %s", this);
    checkNotNull(fileSchema, "No file schema has been initialized for source %s", this);
    if (type == GenericRecord.class) {
        return new GenericDatumReader<>(fileSchema, readSchema);
    } else {
        return new ReflectDatumReader<>(fileSchema, readSchema);
    }
}

Example 43 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project beam by apache.

the class AvroIOTest method testMetadata.

@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
public void testMetadata() throws Exception {
    List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
    File outputFile = tmpFolder.newFile("output.avro");
    p.apply(Create.of(values)).apply(AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding().withMetadata(ImmutableMap.<String, Object>of("stringKey", "stringValue", "longKey", 100L, "bytesKey", "bytesValue".getBytes())));
    p.run();
    DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader());
    assertEquals("stringValue", dataFileStream.getMetaString("stringKey"));
    assertEquals(100L, dataFileStream.getMetaLong("longKey"));
    assertArrayEquals("bytesValue".getBytes(), dataFileStream.getMeta("bytesKey"));
}

Example 44 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project beam by apache.

the class AvroIOTest method testAvroIONullCodecWriteAndReadASingleFile.

@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
public void testAvroIONullCodecWriteAndReadASingleFile() throws Throwable {
    List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
    File outputFile = tmpFolder.newFile("output.avro");
    p.apply(Create.of(values)).apply(AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding().withCodec(CodecFactory.nullCodec()));
    p.run();
    PCollection<GenericClass> input = p.apply(AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath()));
    PAssert.that(input).containsInAnyOrder(values);
    p.run();
    DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader());
    assertEquals("null", dataFileStream.getMetaString("avro.codec"));
}

Example 45 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project samza by apache.

the class AvroFileHdfsReader method open.

@Override
public void open(String pathStr, String singleFileOffset) {
    LOG.info(String.format("%s: Open file [%s] with file offset [%s] for read", systemStreamPartition, pathStr, singleFileOffset));
    Path path = new Path(pathStr);
    try {
        AvroFSInput input = new AvroFSInput(FileContext.getFileContext(path.toUri()), path);
        fileReader = new DataFileReader<>(input, new GenericDatumReader<>());
        seek(singleFileOffset);
    } catch (IOException e) {
        throw new SamzaException(e);
    }
}