Examples with ProtobufBlockWriter com.twitter.elephantbird.mapreduce.io.ProtobufBlockWriter used on opensource projects

Example 1 with ProtobufBlockWriter

use of com.twitter.elephantbird.mapreduce.io.ProtobufBlockWriter in project elephant-bird by twitter.

the class TestProtobufMultiFormatLoader method setUp.

@Before
public void setUp() throws Exception {
    Configuration conf = new Configuration();
    Assume.assumeTrue(CoreTestUtil.okToRunLzoTests(conf));
    pigServer = PigTestUtil.makePigServer();
    inputDir.mkdirs();
    // write to block file
    ProtobufBlockWriter<Person> blk_writer = new ProtobufBlockWriter<Person>(createLzoOut("1-block.lzo", conf), Person.class);
    for (Person rec : records) {
        blk_writer.write(rec);
    }
    blk_writer.close();
    ProtobufWritable<Person> protoWritable = ProtobufWritable.newInstance(Person.class);
    // write tb64 lines
    LzoBinaryB64LineRecordWriter<Person, ProtobufWritable<Person>> b64_writer = LzoBinaryB64LineRecordWriter.newProtobufWriter(Person.class, createLzoOut("2-b64.lzo", conf));
    for (Person rec : records) {
        protoWritable.set(rec);
        b64_writer.write(null, protoWritable);
    }
    b64_writer.close(null);
}