Examples with Record org.apache.avro.generic.GenericData.Record used on opensource projects

Example 36 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class GoogleDriveCopyReader method start.

@Override
public boolean start() throws IOException {
    super.start();
    CopyMode copyMode = properties.copyMode.getValue();
    String source = properties.source.getValue();
    String destinationFolder = properties.destinationFolder.getValue();
    String newName = properties.rename.getValue() ? properties.newName.getValue() : "";
    boolean deleteSourceFile = properties.deleteSourceFile.getValue();
    /* check for destination folder */
    String destinationFolderId = properties.destinationFolderAccessMethod.getValue().equals(AccessMethod.Id) ? destinationFolder : utils.getFolderId(destinationFolder, false);
    /* work on a fileName */
    if (CopyMode.File.equals(copyMode)) {
        /* check for managed resource */
        sourceId = properties.sourceAccessMethod.getValue().equals(AccessMethod.Id) ? source : utils.getFileId(source);
        destinationId = utils.copyFile(sourceId, destinationFolderId, newName, deleteSourceFile);
    } else {
        /* work on a folder */
        /* check for managed resource */
        sourceId = properties.sourceAccessMethod.getValue().equals(AccessMethod.Id) ? source : utils.getFolderId(source, false);
        if (newName.isEmpty()) {
            List<String> paths = utils.getExplodedPath(source);
            newName = paths.get(paths.size() - 1);
        }
        destinationId = utils.copyFolder(sourceId, destinationFolderId, newName);
    }
    // 
    record = new Record(properties.schemaMain.schema.getValue());
    record.put(0, sourceId);
    record.put(1, destinationId);
    result.totalCount++;
    result.successCount++;
    return true;
}

Example 37 with Record

use of org.apache.avro.generic.GenericData.Record in project components by Talend.

the class KeyValueUtils method mergeIndexedRecord.

/**
 * Merge a two IndexedRecords in order to match the outputSchema.
 *
 * @param keyRecord an indexedRecord
 * @param valueRecord an indexedRecord
 * @param outputSchema a schema
 * @return a merged IndexedRecord
 */
public static IndexedRecord mergeIndexedRecord(IndexedRecord keyRecord, IndexedRecord valueRecord, Schema outputSchema) {
    Record outputRecord = new Record(outputSchema);
    Schema keySchema = getUnwrappedSchema(keyRecord);
    Schema valueSchema = getUnwrappedSchema(valueRecord);
    for (Field field : outputSchema.getFields()) {
        if (keySchema.getField(field.name()) != null && valueSchema.getField(field.name()) != null) {
            // The field is present in key and shcema => we are on a record an need to merge its subschema
            Object keyValue = keyRecord.get(keySchema.getField(field.name()).pos());
            Object valueValue = valueRecord.get(valueSchema.getField(field.name()).pos());
            if (keyValue instanceof Record) {
                Schema keyChildSchema = getUnwrappedSchema(keySchema.getField(field.name()));
                Schema valueChildSchema = getUnwrappedSchema(valueSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (keyChildSchema.getType().equals(Type.RECORD) && valueChildSchema.getType().equals(Type.RECORD) && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = mergeIndexedRecord((IndexedRecord) keyValue, (IndexedRecord) valueValue, outputChildSchema);
                    outputRecord.put(field.name(), childRecord);
                }
            } else {
                if (keyValue != null || AvroUtils.isNullable(field.schema())) {
                    outputRecord.put(field.name(), keyValue);
                }
            }
        } else if (keySchema.getField(field.name()) != null) {
            Object keyValue = keyRecord.get(keySchema.getField(field.name()).pos());
            if (keyValue != null || AvroUtils.isNullable(field.schema())) {
                outputRecord.put(field.name(), keyValue);
            }
        } else if (valueSchema.getField(field.name()) != null) {
            Object valueValue = valueRecord.get(valueSchema.getField(field.name()).pos());
            if (valueValue != null || AvroUtils.isNullable(field.schema())) {
                outputRecord.put(field.name(), valueValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.put(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord;
}

Example 38 with Record

use of org.apache.avro.generic.GenericData.Record in project drill by axbaretto.

the class KafkaMessageGenerator method populateAvroMsgIntoKafka.

public void populateAvroMsgIntoKafka(String topic, int numMsg) throws IOException {
    KafkaProducer<String, GenericRecord> producer = new KafkaProducer<String, GenericRecord>(producerProperties);
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(Resources.getResource("drill-avro-test.avsc").openStream());
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    Random rand = new Random();
    for (int i = 0; i < numMsg; ++i) {
        builder.set("key1", UUID.randomUUID().toString());
        builder.set("key2", rand.nextInt());
        builder.set("key3", rand.nextBoolean());
        List<Integer> list = Lists.newArrayList();
        list.add(rand.nextInt(100));
        list.add(rand.nextInt(100));
        list.add(rand.nextInt(100));
        builder.set("key5", list);
        Map<String, Double> map = Maps.newHashMap();
        map.put("key61", rand.nextDouble());
        map.put("key62", rand.nextDouble());
        builder.set("key6", map);
        Record producerRecord = builder.build();
        ProducerRecord<String, GenericRecord> record = new ProducerRecord<String, GenericRecord>(topic, producerRecord);
        producer.send(record);
    }
    producer.close();
}

Example 39 with Record

use of org.apache.avro.generic.GenericData.Record in project drill by apache.

the class KafkaMessageGenerator method populateAvroMsgIntoKafka.

public void populateAvroMsgIntoKafka(String topic, int numMsg) {
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
    try (KafkaProducer<Object, GenericRecord> producer = new KafkaProducer<>(producerProperties)) {
        Schema.Parser parser = new Schema.Parser();
        String userSchema = "{\"type\":\"record\"," + "\"name\":\"myrecord\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}," + "{\"name\":\"key2\",\"type\":\"int\"}," + "{\"name\":\"key3\",\"type\":\"boolean\"}," + "{\"name\":\"key5\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}," + "{\"name\":\"key6\",\"type\":{\"type\":\"record\",\"name\":\"myrecord6\",\"fields\":[" + "{\"name\":\"key61\",\"type\":\"double\"}," + "{\"name\":\"key62\",\"type\":\"double\"}]}}]}";
        Schema valueSchema = parser.parse(userSchema);
        GenericRecordBuilder valueBuilder = new GenericRecordBuilder(valueSchema);
        String key1Schema = "{\"type\":\"record\"," + "\"name\":\"key1record\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}]}\"";
        Schema keySchema = parser.parse(key1Schema);
        GenericRecordBuilder keyBuilder = new GenericRecordBuilder(keySchema);
        Random rand = new Random();
        for (int i = 0; i < numMsg; ++i) {
            // value record
            String key1 = UUID.randomUUID().toString();
            valueBuilder.set("key1", key1);
            valueBuilder.set("key2", rand.nextInt());
            valueBuilder.set("key3", rand.nextBoolean());
            List<Integer> list = Lists.newArrayList();
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            valueBuilder.set("key5", list);
            GenericRecordBuilder innerBuilder = new GenericRecordBuilder(valueSchema.getField("key6").schema());
            innerBuilder.set("key61", rand.nextDouble());
            innerBuilder.set("key62", rand.nextDouble());
            valueBuilder.set("key6", innerBuilder.build());
            Record producerRecord = valueBuilder.build();
            // key record
            keyBuilder.set("key1", key1);
            Record keyRecord = keyBuilder.build();
            ProducerRecord<Object, GenericRecord> record = new ProducerRecord<>(topic, keyRecord, producerRecord);
            producer.send(record);
        }
    }
}

Example 40 with Record

use of org.apache.avro.generic.GenericData.Record in project beam by apache.

the class BigQueryIOStorageReadTest method createRecord.

private static GenericRecord createRecord(String name, long number, Schema schema) {
    GenericRecord genericRecord = new Record(schema);
    genericRecord.put("name", name);
    genericRecord.put("number", number);
    return genericRecord;
}