Examples with Record org.apache.avro.generic.GenericData.Record used on opensource projects

Example 1 with Record

use of org.apache.avro.generic.GenericData.Record in project crunch by cloudera.

the class AvroDeepCopierTest method testDeepCopyGeneric.

@Test
public void testDeepCopyGeneric() {
    Record record = new Record(Person.SCHEMA$);
    record.put("name", "John Doe");
    record.put("age", 42);
    record.put("siblingnames", Lists.newArrayList());
    Record deepCopyRecord = new AvroDeepCopier.AvroGenericDeepCopier(Person.SCHEMA$).deepCopy(record);
    assertEquals(record, deepCopyRecord);
    assertNotSame(record, deepCopyRecord);
}

Example 2 with Record

use of org.apache.avro.generic.GenericData.Record in project crunch by cloudera.

the class AvroFileSourceTargetTest method testGeneric.

@Test
public void testGeneric() throws IOException {
    String genericSchemaJson = Person.SCHEMA$.toString().replace("Person", "GenericPerson");
    Schema genericPersonSchema = new Schema.Parser().parse(genericSchemaJson);
    GenericRecord savedRecord = new GenericData.Record(genericPersonSchema);
    savedRecord.put("name", "John Doe");
    savedRecord.put("age", 42);
    savedRecord.put("siblingnames", Lists.newArrayList("Jimmy", "Jane"));
    populateGenericFile(Lists.newArrayList(savedRecord), genericPersonSchema);
    Pipeline pipeline = new MRPipeline(AvroFileSourceTargetTest.class);
    PCollection<Record> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(), Avros.generics(genericPersonSchema)));
    List<Record> recordList = Lists.newArrayList(genericCollection.materialize());
    assertEquals(Lists.newArrayList(savedRecord), Lists.newArrayList(recordList));
}

Example 3 with Record

use of org.apache.avro.generic.GenericData.Record in project nifi by apache.

the class TestKiteProcessorsCluster method testBasicStoreToHive.

@Test
public void testBasicStoreToHive() throws IOException {
    String datasetUri = "dataset:hive:ns/test";
    Dataset<Record> dataset = Datasets.create(datasetUri, descriptor, Record.class);
    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.assertNotValid();
    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();
    List<Record> users = Lists.newArrayList(user("a", "a@example.com"), user("b", "b@example.com"), user("c", "c@example.com"));
    runner.enqueue(streamFor(users));
    runner.run();
    runner.assertAllFlowFilesTransferred("success", 1);
    List<Record> stored = Lists.newArrayList((Iterable<Record>) dataset.newReader());
    Assert.assertEquals("Records should match", users, stored);
    Datasets.delete(datasetUri);
}

Example 4 with Record

use of org.apache.avro.generic.GenericData.Record in project nifi by apache.

the class TestKiteStorageProcessor method testIncompatibleSchema.

@Test
public void testIncompatibleSchema() throws IOException {
    Schema incompatible = SchemaBuilder.record("User").fields().requiredLong("id").requiredString("username").optionalString(// the dataset requires this field
    "email").endRecord();
    // this user has the email field and could be stored, but the schema is
    // still incompatible so the entire stream is rejected
    Record incompatibleUser = new Record(incompatible);
    incompatibleUser.put("id", 1L);
    incompatibleUser.put("username", "a");
    incompatibleUser.put("email", "a@example.com");
    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();
    runner.enqueue(streamFor(incompatibleUser));
    runner.run();
    runner.assertAllFlowFilesTransferred("incompatible", 1);
}

Example 5 with Record

use of org.apache.avro.generic.GenericData.Record in project nifi by apache.

the class TestUtil method bytesFor.

private static byte[] bytesFor(List<Record> records) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataFileWriter<Record> writer = new DataFileWriter<>(AvroUtil.newDatumWriter(records.get(0).getSchema(), Record.class));
    writer.setCodec(CodecFactory.snappyCodec());
    writer = writer.create(records.get(0).getSchema(), out);
    for (Record record : records) {
        writer.append(record);
    }
    writer.flush();
    return out.toByteArray();
}