Examples with GenericArray org.apache.avro.generic.GenericArray used on opensource projects

Example 11 with GenericArray

use of org.apache.avro.generic.GenericArray in project jvm-serializers by eishay.

the class AvroTransformer method forward.

// ----------------------------------------------------------
// Forward
public MediaContent forward(data.media.MediaContent mc) {
    GenericArray<Image> images = new GenericData.Array<Image>(mc.images.size(), Avro.Media.sImages);
    for (data.media.Image image : mc.images) {
        images.add(forwardImage(image));
    }
    MediaContent amc = new MediaContent();
    amc.setMedia(forwardMedia(mc.media));
    amc.setImages(images);
    return amc;
}

Example 12 with GenericArray

use of org.apache.avro.generic.GenericArray in project hive by apache.

the class TestAvroSerializer method canSerializeLists.

@Test
public void canSerializeLists() throws SerDeException, IOException {
    List<Integer> intList = new ArrayList<Integer>();
    Collections.addAll(intList, 1, 2, 3);
    String field = "{ \"name\":\"list1\", \"type\":{\"type\":\"array\", \"items\":\"int\"} }";
    GenericRecord r = serializeAndDeserialize(field, "list1", intList);
    final Object list1 = r.get("list1");
    Assert.assertTrue(list1 instanceof GenericArray);
    Assert.assertTrue(list1 instanceof List);
    assertEquals(intList, list1);
}

Example 13 with GenericArray

use of org.apache.avro.generic.GenericArray in project databus by linkedin.

the class BootstrapAuditTester method compareField.

private boolean compareField(Field f, Object databaseFieldValue, Object avroField) {
    // NULL condition handled
    if (databaseFieldValue == avroField) {
        return true;
    }
    if (databaseFieldValue == null) {
        // avroField cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " null databaseFieldValue but non-null avroField ");
        return false;
    }
    if (avroField == null) {
        // databaseFieldValue cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " non-null databaseFieldValue but null avroField ");
        return false;
    }
    try {
        // == f.schema() if f is not a union
        Schema fieldSchema = SchemaHelper.unwindUnionSchema(f);
        Type avroFieldType = fieldSchema.getType();
        if (_sDebug) {
            LOG.debug("Checking for type:" + avroFieldType + ", Field:" + f.name() + ", Exp:" + databaseFieldValue + ", Got:" + avroField);
        }
        switch(avroFieldType) {
            case BOOLEAN:
                assertEquals(f.name(), databaseFieldValue, avroField);
                break;
            case BYTES:
                byte[] byteArr = null;
                if (databaseFieldValue instanceof Blob) {
                    Blob b = (Blob) databaseFieldValue;
                    byteArr = b.getBytes(1, (int) b.length());
                } else {
                    byteArr = (byte[]) databaseFieldValue;
                }
                assertEquals(f.name(), byteArr, avroField);
                break;
            case DOUBLE:
                assertEquals(f.name(), new Double(((Number) databaseFieldValue).doubleValue()), (avroField));
                break;
            case FLOAT:
                assertEquals(f.name(), new Float(((Number) databaseFieldValue).floatValue()), (avroField));
                break;
            case INT:
                assertEquals(f.name(), Integer.valueOf(((Number) databaseFieldValue).intValue()), (avroField));
                break;
            case LONG:
                if (databaseFieldValue instanceof Number) {
                    long lvalue = ((Number) databaseFieldValue).longValue();
                    assertEquals(f.name(), lvalue, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Timestamp) {
                    long time = ((Timestamp) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Date) {
                    long time = ((Date) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else {
                    Class timestampClass = null, dateClass = null;
                    try {
                        timestampClass = OracleJarUtils.loadClass("oracle.sql.TIMESTAMP");
                        dateClass = OracleJarUtils.loadClass("oracle.sql.DATE");
                    } catch (Exception e) {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long. Unable to get Oracle datatypes " + e.getMessage();
                        LOG.error(errMsg);
                        throw new EventCreationException(errMsg);
                    }
                    if (timestampClass.isInstance(databaseFieldValue)) {
                        try {
                            Object tsc = timestampClass.cast(databaseFieldValue);
                            Method dateValueMethod = timestampClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(tsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.TIMESTAMP value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else if (dateClass.isInstance(databaseFieldValue)) {
                        try {
                            Object dsc = dateClass.cast(databaseFieldValue);
                            Method dateValueMethod = dateClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(dsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.DATE value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long for field " + f.name();
                        LOG.error(errMsg);
                        throw new RuntimeException();
                    }
                }
                break;
            case STRING:
                if (databaseFieldValue instanceof Clob) {
                    String text = null;
                    try {
                        text = OracleAvroGenericEventFactory.extractClobText((Clob) databaseFieldValue, f.name());
                    } catch (EventCreationException ex) {
                        LOG.error("compareField error: " + ex.getMessage(), ex);
                    }
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                } else {
                    String text = databaseFieldValue.toString();
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                }
                break;
            case NULL:
                assertNull(f.name(), databaseFieldValue);
                assertNull(f.name(), avroField);
                break;
            case ARRAY:
                GenericArray<GenericRecord> avroArray = (GenericArray<GenericRecord>) avroField;
                Schema elementSchema = fieldSchema.getElementType();
                Array array = (Array) databaseFieldValue;
                ResultSet arrayResultSet = array.getResultSet();
                int i = 0;
                while (arrayResultSet.next()) {
                    // Get the underlying structure from the database. Oracle returns the structure in the
                    // second column of the array's ResultSet
                    Struct struct = (Struct) arrayResultSet.getObject(2);
                    Object[] attributes = struct.getAttributes();
                    GenericRecord avroElement = avroArray.get(i++);
                    // have to use dbFieldPosition recorded in the schema definition.
                    for (Field field : elementSchema.getFields()) {
                        int dbFieldPosition = Integer.valueOf(SchemaHelper.getMetaField(field, "dbFieldPosition"));
                        Object dbFieldValue = attributes[dbFieldPosition];
                        Object avroFieldValue = avroElement.get(field.name());
                        compareField(field, dbFieldValue, avroFieldValue);
                    }
                }
                break;
            case RECORD:
                assert (compareRecord(fieldSchema, (Struct) databaseFieldValue, (GenericRecord) avroField)) : "comparison of Avro 'record' type failed";
                break;
            case ENUM:
            case FIXED:
            case MAP:
            case UNION:
            default:
                String msg = "Audit for these fields not yet implemented for: " + fieldSchema.getName() + ", Avro type: " + avroFieldType;
                LOG.error(msg);
                throw new RuntimeException(msg);
        }
    } catch (AssertionError err) {
        LOG.error("compareField error: " + err.getMessage() + " field= " + f.name());
        return false;
    } catch (ClassCastException ce) {
        LOG.error("compareField error: " + ce.getMessage() + " field=" + f.name(), ce);
        return false;
    } catch (Exception ex) {
        LOG.error("compareField error: " + ex.getMessage() + " field=" + f.name(), ex);
        return false;
    }
    return true;
}

Example 14 with GenericArray

use of org.apache.avro.generic.GenericArray in project trevni by cutting.

the class RandomData method generate.

@SuppressWarnings(value = "unchecked")
private static Object generate(Schema schema, Random random, int d) {
    switch(schema.getType()) {
        case RECORD:
            GenericRecord record = new GenericData.Record(schema);
            for (Schema.Field field : schema.getFields()) record.put(field.name(), generate(field.schema(), random, d + 1));
            return record;
        case ENUM:
            List<String> symbols = schema.getEnumSymbols();
            return new GenericData.EnumSymbol(schema, symbols.get(random.nextInt(symbols.size())));
        case ARRAY:
            int length = (random.nextInt(5) + 2) - d;
            GenericArray<Object> array = new GenericData.Array(length <= 0 ? 0 : length, schema);
            for (int i = 0; i < length; i++) array.add(generate(schema.getElementType(), random, d + 1));
            return array;
        case MAP:
            length = (random.nextInt(5) + 2) - d;
            Map<Object, Object> map = new HashMap<Object, Object>(length <= 0 ? 0 : length);
            for (int i = 0; i < length; i++) {
                map.put(TestUtil.randomString(random), generate(schema.getValueType(), random, d + 1));
            }
            return map;
        case UNION:
            List<Schema> types = schema.getTypes();
            return generate(types.get(random.nextInt(types.size())), random, d);
        case FIXED:
            byte[] bytes = new byte[schema.getFixedSize()];
            random.nextBytes(bytes);
            return new GenericData.Fixed(schema, bytes);
        case STRING:
            return TestUtil.randomString(random);
        case BYTES:
            return TestUtil.randomBytes(random);
        case INT:
            return random.nextInt();
        case LONG:
            return random.nextLong();
        case FLOAT:
            return random.nextFloat();
        case DOUBLE:
            return random.nextDouble();
        case BOOLEAN:
            return random.nextBoolean();
        case NULL:
            return null;
        default:
            throw new RuntimeException("Unknown type: " + schema);
    }
}

Example 15 with GenericArray

use of org.apache.avro.generic.GenericArray in project core by s4.

the class AvroSerDeser method copyRecord.

public static GenericRecord copyRecord(Map<String, Object> record, Schema schema, GenericRecord avroRecord) {
    Map<String, Field> fieldMap = schema.getFields();
    for (String fieldName : record.keySet()) {
        Field field = fieldMap.get(fieldName);
        if (field == null) {
            // not in schema, ignore
            continue;
        }
        Schema fieldSchema = field.schema();
        if (fieldSchema.getType().equals(Schema.Type.UNION)) {
            List<Schema> actualSchemas = fieldSchema.getTypes();
            // this should only contain two entries, the actual type and
            // NULL
            fieldSchema = null;
            for (Schema actualSchema : actualSchemas) {
                if (actualSchema.getType().equals(Schema.Type.NULL)) {
                    continue;
                }
                fieldSchema = actualSchema;
                break;
            }
            if (fieldSchema == null) {
                return avroRecord;
            }
        }
        Object value = record.get(fieldName);
        if (value == null) {
            continue;
        }
        if (fieldSchema.getType().equals(Schema.Type.STRING)) {
            avroRecord.put(fieldName, new Utf8((String) value));
        } else if (fieldSchema.getType().equals(Schema.Type.RECORD)) {
            if (!(value instanceof Map)) {
                // schema mismatch
                continue;
            }
            avroRecord.put(fieldName, copyRecord((Map<String, Object>) record.get(fieldName), fieldSchema, new GenericData.Record(fieldSchema)));
        } else if (fieldSchema.getType().equals(Schema.Type.ARRAY)) {
            if (!(value instanceof List)) {
                // schema mismatch
                continue;
            }
            List<Map<String, Object>> list = (List<Map<String, Object>>) record.get(fieldName);
            GenericArray<GenericRecord> avroArray = new GenericData.Array<GenericRecord>(list.size());
            avroRecord.put(fieldName, avroArray);
            copyArray(list, fieldSchema.getElementType(), avroArray);
        } else if (fieldSchema.getType().equals(Schema.Type.ENUM) || fieldSchema.getType().equals(Schema.Type.UNION) || fieldSchema.getType().equals(Schema.Type.BYTES) || fieldSchema.getType().equals(Schema.Type.MAP)) {
            // we don't properly handle that right now
            continue;
        } else {
            avroRecord.put(fieldName, value);
        }
    }
    return avroRecord;
}