Examples with Type org.apache.avro.Schema.Type used on opensource projects

Example 51 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class OracleAvroGenericEventFactory method put.

private void put(GenericRecord record, Field field, Object databaseFieldValue) throws EventCreationException {
    // Get the field name and type from the event schema
    String schemaFieldName = field.name();
    // == field.schema() if not a union
    Schema fieldSchema = SchemaHelper.unwindUnionSchema(field);
    Type avroFieldType = fieldSchema.getType();
    if (databaseFieldValue == null) {
        // The field value was null. If the field is nullable then we do nothing. Otherwise this is an error.
        boolean isNullAllowedInSchema = SchemaHelper.isNullable(field);
        if (!isNullAllowedInSchema) {
            throw new EventCreationException("Null value not allowed for field " + schemaFieldName);
        }
    } else {
        if (_log.isTraceEnabled()) {
            _log.trace("record.put(\"" + schemaFieldName + "\", (" + avroFieldType + ") \"" + databaseFieldValue + "\"");
        }
        try {
            switch(avroFieldType) {
                case BOOLEAN:
                case BYTES:
                case DOUBLE:
                case FLOAT:
                case INT:
                case LONG:
                case STRING:
                case NULL:
                    putSimpleValue(record, schemaFieldName, avroFieldType, databaseFieldValue);
                    break;
                case RECORD:
                    addOracleRecordToParent(record, schemaFieldName, fieldSchema, (Struct) databaseFieldValue);
                    break;
                case ARRAY:
                    putArray(record, schemaFieldName, fieldSchema, (Array) databaseFieldValue);
                    break;
                // exists in some Espresso schemas:  don't blindly cut and paste!
                case ENUM:
                // ditto
                case MAP:
                case FIXED:
                // shouldn't be possible, given unwindUnionSchema() call above
                case UNION:
                default:
                    throw new EventCreationException("Don't know how to populate this type of field: " + avroFieldType);
            }
        } catch (ClassCastException ex) {
            throw new EventCreationException("Type conversion error for field name (" + field.name() + ") in source " + _sourceId + ". Value was: " + databaseFieldValue + " avro field was: " + avroFieldType, ex);
        }
    }
}

Example 52 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class BootstrapAuditMain method main.

/**
 * @param args
 */
public static void main(String[] args) throws Exception {
    BootstrapSeederMain.init(args);
    BootstrapSeederMain.StaticConfig staticConfig = BootstrapSeederMain.getStaticConfig();
    int interval = staticConfig.getController().getCommitInterval();
    int sourceChunkSize = staticConfig.getController().getNumRowsPerQuery();
    List<OracleTriggerMonitoredSourceInfo> sources = BootstrapSeederMain.getSources();
    BootstrapDBSeeder seeder = BootstrapSeederMain.getSeeder();
    BootstrapSrcDBEventReader seedController = BootstrapSeederMain.getReader();
    Map<String, String> pKeyNameMap = seedController.getpKeyNameMap();
    Map<String, DbusEventKey.KeyType> pKeyTypeMap = seedController.getpKeyTypeMap();
    for (OracleTriggerMonitoredSourceInfo source : sources) {
        short srcId = source.getSourceId();
        new ConcurrentHashMap<Long, ResultSetEntry>();
        OracleTableReader oracleReader = null;
        MySQLTableReader mySQLReader = null;
        try {
            SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(staticConfig.getSchemaRegistry().getFileSystem());
            Map<Short, String> schemaSet = schemaRegistry.fetchAllSchemaVersionsBySourceName(source.getSourceName());
            VersionedSchemaSet vSchemaSet = new VersionedSchemaSet();
            Iterator<Map.Entry<Short, String>> it = schemaSet.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Short, String> pairs = it.next();
                Schema s = Schema.parse(pairs.getValue());
                VersionedSchema vs = new VersionedSchema(s.getFullName(), pairs.getKey(), s, null);
                vSchemaSet.add(vs);
            }
            /* Try and identify the schema key */
            VersionedSchema vschema = schemaRegistry.fetchLatestVersionedSchemaBySourceName(source.getSourceName());
            Schema schema = Schema.parse(vschema.getSchema().toString());
            LOG.info("Schema =" + vschema.getSchema() + "version=" + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());
            /* Determine type of field txn */
            Field txnFieldType = schema.getField("txn");
            if (txnFieldType == null) {
                throw new Exception("Unable to find field called 'txn'. Cannot proceeed\n");
            }
            Type txnType = SchemaHelper.getAnyType(txnFieldType);
            /*
         * Determine primary key of schema. This is assumed to be invariant
         * across versions
         */
            String keyOverrideName = SchemaHelper.getMetaField(schema, "pk");
            String keyColumnName = "key";
            if (null != keyOverrideName) {
                keyColumnName = keyOverrideName;
            }
            Field pkeyField = schema.getField(keyColumnName);
            if (null == pkeyField) {
                keyColumnName = "id";
                pkeyField = schema.getField("id");
            }
            if (null == pkeyField) {
                throw new Exception("Unable to get the primary key for schema. Schema is :" + schema);
            }
            DbusEventAvroDecoder decoder = new DbusEventAvroDecoder(vSchemaSet);
            BootstrapAuditTester auditor = new BootstrapAuditTester(schema, BootstrapSrcDBEventReader.getTableName(source));
            List<BootstrapAuditTester> auditors = new ArrayList<BootstrapAuditTester>();
            auditors.add(auditor);
            oracleReader = new OracleTableReader(BootstrapSeederMain.getDataStore().getConnection(), BootstrapSrcDBEventReader.getTableName(source), pkeyField, SchemaHelper.getMetaField(pkeyField, "dbFieldName"), SchemaHelper.getAnyType(pkeyField), sourceChunkSize, seedController.getPKIndex(source), seedController.getQueryHint(source));
            mySQLReader = new MySQLTableReader(seeder.getConnection(), // THis is the primary
            seeder.getTableName(srcId), // THis is the primary
            pkeyField, // THis is the primary
            "id", // bootstrapDB
            SchemaHelper.getAnyType(pkeyField), interval);
            double samplePct = BootstrapSeederMain.getValidationSamplePct();
            TableComparator comparator = new TableComparator(oracleReader, mySQLReader, auditor, decoder, interval, pKeyNameMap.get(source.getEventView()), pKeyTypeMap.get(source.getEventView()), txnType, samplePct);
            boolean success = false;
            if (BootstrapSeederMain.getValidationType().equals("point")) {
                success = comparator.compareRecordsPoint();
            } else if (BootstrapSeederMain.getValidationType().equals("pointBs")) {
                success = comparator.compareRecordsPointBs();
            } else {
                success = comparator.compareRecordsNew();
            }
            if (success)
                LOG.info("Audit completed successfully");
            else
                LOG.error("Audit FAILED !!! ");
        } catch (Exception ex) {
            LOG.error("Caught an exception ex", ex);
            throw ex;
        } finally {
            if (null != oracleReader)
                oracleReader.close();
        }
    }
    DBHelper.close(seeder.getConnection());
}

Example 53 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class BootstrapAuditTester method compareField.

private boolean compareField(Field f, Object databaseFieldValue, Object avroField) {
    // NULL condition handled
    if (databaseFieldValue == avroField) {
        return true;
    }
    if (databaseFieldValue == null) {
        // avroField cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " null databaseFieldValue but non-null avroField ");
        return false;
    }
    if (avroField == null) {
        // databaseFieldValue cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " non-null databaseFieldValue but null avroField ");
        return false;
    }
    try {
        // == f.schema() if f is not a union
        Schema fieldSchema = SchemaHelper.unwindUnionSchema(f);
        Type avroFieldType = fieldSchema.getType();
        if (_sDebug) {
            LOG.debug("Checking for type:" + avroFieldType + ", Field:" + f.name() + ", Exp:" + databaseFieldValue + ", Got:" + avroField);
        }
        switch(avroFieldType) {
            case BOOLEAN:
                assertEquals(f.name(), databaseFieldValue, avroField);
                break;
            case BYTES:
                byte[] byteArr = null;
                if (databaseFieldValue instanceof Blob) {
                    Blob b = (Blob) databaseFieldValue;
                    byteArr = b.getBytes(1, (int) b.length());
                } else {
                    byteArr = (byte[]) databaseFieldValue;
                }
                assertEquals(f.name(), byteArr, avroField);
                break;
            case DOUBLE:
                assertEquals(f.name(), new Double(((Number) databaseFieldValue).doubleValue()), (avroField));
                break;
            case FLOAT:
                assertEquals(f.name(), new Float(((Number) databaseFieldValue).floatValue()), (avroField));
                break;
            case INT:
                assertEquals(f.name(), Integer.valueOf(((Number) databaseFieldValue).intValue()), (avroField));
                break;
            case LONG:
                if (databaseFieldValue instanceof Number) {
                    long lvalue = ((Number) databaseFieldValue).longValue();
                    assertEquals(f.name(), lvalue, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Timestamp) {
                    long time = ((Timestamp) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Date) {
                    long time = ((Date) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else {
                    Class timestampClass = null, dateClass = null;
                    try {
                        timestampClass = OracleJarUtils.loadClass("oracle.sql.TIMESTAMP");
                        dateClass = OracleJarUtils.loadClass("oracle.sql.DATE");
                    } catch (Exception e) {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long. Unable to get Oracle datatypes " + e.getMessage();
                        LOG.error(errMsg);
                        throw new EventCreationException(errMsg);
                    }
                    if (timestampClass.isInstance(databaseFieldValue)) {
                        try {
                            Object tsc = timestampClass.cast(databaseFieldValue);
                            Method dateValueMethod = timestampClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(tsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.TIMESTAMP value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else if (dateClass.isInstance(databaseFieldValue)) {
                        try {
                            Object dsc = dateClass.cast(databaseFieldValue);
                            Method dateValueMethod = dateClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(dsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.DATE value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long for field " + f.name();
                        LOG.error(errMsg);
                        throw new RuntimeException();
                    }
                }
                break;
            case STRING:
                if (databaseFieldValue instanceof Clob) {
                    String text = null;
                    try {
                        text = OracleAvroGenericEventFactory.extractClobText((Clob) databaseFieldValue, f.name());
                    } catch (EventCreationException ex) {
                        LOG.error("compareField error: " + ex.getMessage(), ex);
                    }
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                } else {
                    String text = databaseFieldValue.toString();
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                }
                break;
            case NULL:
                assertNull(f.name(), databaseFieldValue);
                assertNull(f.name(), avroField);
                break;
            case ARRAY:
                GenericArray<GenericRecord> avroArray = (GenericArray<GenericRecord>) avroField;
                Schema elementSchema = fieldSchema.getElementType();
                Array array = (Array) databaseFieldValue;
                ResultSet arrayResultSet = array.getResultSet();
                int i = 0;
                while (arrayResultSet.next()) {
                    // Get the underlying structure from the database. Oracle returns the structure in the
                    // second column of the array's ResultSet
                    Struct struct = (Struct) arrayResultSet.getObject(2);
                    Object[] attributes = struct.getAttributes();
                    GenericRecord avroElement = avroArray.get(i++);
                    // have to use dbFieldPosition recorded in the schema definition.
                    for (Field field : elementSchema.getFields()) {
                        int dbFieldPosition = Integer.valueOf(SchemaHelper.getMetaField(field, "dbFieldPosition"));
                        Object dbFieldValue = attributes[dbFieldPosition];
                        Object avroFieldValue = avroElement.get(field.name());
                        compareField(field, dbFieldValue, avroFieldValue);
                    }
                }
                break;
            case RECORD:
                assert (compareRecord(fieldSchema, (Struct) databaseFieldValue, (GenericRecord) avroField)) : "comparison of Avro 'record' type failed";
                break;
            case ENUM:
            case FIXED:
            case MAP:
            case UNION:
            default:
                String msg = "Audit for these fields not yet implemented for: " + fieldSchema.getName() + ", Avro type: " + avroFieldType;
                LOG.error(msg);
                throw new RuntimeException(msg);
        }
    } catch (AssertionError err) {
        LOG.error("compareField error: " + err.getMessage() + " field= " + f.name());
        return false;
    } catch (ClassCastException ce) {
        LOG.error("compareField error: " + ce.getMessage() + " field=" + f.name(), ce);
        return false;
    } catch (Exception ex) {
        LOG.error("compareField error: " + ex.getMessage() + " field=" + f.name(), ex);
        return false;
    }
    return true;
}

Example 54 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class SchemaHelper method unwindUnionSchema.

/**
 * @return the field's schema; if the field is a union, the schema for the first non-null type from the
 *         union is returned.
 */
public static Schema unwindUnionSchema(Field field) {
    Schema schema = field.schema();
    Type fieldType = schema.getType();
    // If this is a union, check the child types and return the first non-null schema
    if (fieldType == Type.UNION) {
        List<Schema> unionTypes = schema.getTypes();
        for (Schema unionSubSchema : unionTypes) {
            if (unionSubSchema.getType() != Type.NULL) {
                return unionSubSchema;
            }
        }
    }
    return schema;
}

Example 55 with Type

use of org.apache.avro.Schema.Type in project bunsen by cerner.

the class DefinitionToAvroVisitor method visitChoice.

@Override
public HapiConverter<Schema> visitChoice(String elementName, Map<String, HapiConverter<Schema>> choiceTypes) {
    List<Field> fields = choiceTypes.entrySet().stream().map(entry -> {
        // Ensure first character of the field is lower case.
        String fieldName = lowercase(entry.getKey());
        return new Field(fieldName, nullable(entry.getValue().getDataType()), "Choice field", JsonProperties.NULL_VALUE);
    }).collect(Collectors.toList());
    String fieldTypesString = choiceTypes.entrySet().stream().map(choiceEntry -> {
        // References need their full record name, which includes the permissible referent types
        if (choiceEntry.getKey().equals("Reference")) {
            return choiceEntry.getValue().getDataType().getName();
        } else {
            return choiceEntry.getKey();
        }
    }).sorted().map(StringUtils::capitalize).collect(Collectors.joining());
    String fullName = basePackage + "." + "Choice" + fieldTypesString;
    HapiConverter<Schema> converter = visitedConverters.get(fullName);
    if (converter == null) {
        Schema schema = Schema.createRecord("Choice" + fieldTypesString, "Structure for FHIR choice type ", basePackage, false, fields);
        converter = new HapiChoiceToAvroConverter(choiceTypes, schema, fhirSupport);
        visitedConverters.put(fullName, converter);
    }
    return converter;
}