Examples with GenericRecord org.apache.avro.generic.GenericRecord used on opensource projects

Example 31 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project haivvreo by jghoman.

the class AvroContainerOutputFormat method getHiveRecordWriter.

@Override
public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException {
    Schema schema;
    try {
        schema = HaivvreoUtils.determineSchemaOrThrowException(jobConf, properties);
    } catch (HaivvreoException e) {
        throw new IOException(e);
    }
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
    if (isCompressed) {
        int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
        CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName);
        dfw.setCodec(factory);
    }
    dfw.create(schema, path.getFileSystem(jobConf).create(path));
    return new AvroGenericRecordWriter(dfw);
}

Example 32 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project haivvreo by jghoman.

the class TestAvroSerializer method canSerializeUnions.

@Test
public void canSerializeUnions() throws SerDeException, IOException {
    String field = "{ \"name\":\"union1\", \"type\":[\"float\", \"boolean\", \"string\"] }";
    GenericRecord r = serializeAndDeserialize(field, "union1", 424.4f);
    assertEquals(424.4f, r.get("union1"));
    r = serializeAndDeserialize(field, "union1", true);
    assertEquals(true, r.get("union1"));
    r = serializeAndDeserialize(field, "union1", "hello");
    assertEquals("hello", r.get("union1"));
}

Example 33 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project haivvreo by jghoman.

the class TestAvroSerializer method canSerializeBytes.

@Test
public void canSerializeBytes() throws SerDeException, IOException {
    String field = "{ \"name\":\"bytes1\", \"type\":\"bytes\" }";
    ByteBuffer bb = ByteBuffer.wrap("easy as one two three".getBytes());
    bb.rewind();
    GenericRecord r = serializeAndDeserialize(field, "bytes1", bb);
    assertEquals(bb, r.get("bytes1"));
}

Example 34 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project haivvreo by jghoman.

the class TestAvroSerializer method canSerializeNullableTypes.

@Test
public void canSerializeNullableTypes() throws SerDeException, IOException {
    String field = "{ \"name\":\"nullableint\", \"type\":[\"int\", \"null\"] }";
    GenericRecord r = serializeAndDeserialize(field, "nullableint", 42);
    assertEquals(42, r.get("nullableint"));
    r = serializeAndDeserialize(field, "nullableint", null);
    assertNull(r.get("nullableint"));
}

Example 35 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project haivvreo by jghoman.

the class TestAvroSerializer method canSerializeLists.

@Test
public void canSerializeLists() throws SerDeException, IOException {
    List<Integer> intList = new ArrayList<Integer>();
    Collections.addAll(intList, 1, 2, 3);
    String field = "{ \"name\":\"list1\", \"type\":{\"type\":\"array\", \"items\":\"int\"} }";
    GenericRecord r = serializeAndDeserialize(field, "list1", intList);
    assertEquals(intList, r.get("list1"));
}