Examples with Type org.apache.avro.Schema.Type used on opensource projects

Example 16 with Type

use of org.apache.avro.Schema.Type in project gora by apache.

the class SolrStore method getUnionSchema.

/**
   * Given an object and the object schema this function obtains, from within
   * the UNION schema, the position of the type used. If no data type can be
   * inferred then we return a default value of position 0.
   * 
   * @param pValue
   * @param pUnionSchema
   * @return the unionSchemaPosition.
   */
private int getUnionSchema(Object pValue, Schema pUnionSchema) {
    int unionSchemaPos = 0;
    for (Schema currentSchema : pUnionSchema.getTypes()) {
        Type schemaType = currentSchema.getType();
        if (pValue instanceof Utf8 && schemaType.equals(Type.STRING))
            return unionSchemaPos;
        else if (pValue instanceof ByteBuffer && schemaType.equals(Type.BYTES))
            return unionSchemaPos;
        else if (pValue instanceof Integer && schemaType.equals(Type.INT))
            return unionSchemaPos;
        else if (pValue instanceof Long && schemaType.equals(Type.LONG))
            return unionSchemaPos;
        else if (pValue instanceof Double && schemaType.equals(Type.DOUBLE))
            return unionSchemaPos;
        else if (pValue instanceof Float && schemaType.equals(Type.FLOAT))
            return unionSchemaPos;
        else if (pValue instanceof Boolean && schemaType.equals(Type.BOOLEAN))
            return unionSchemaPos;
        else if (pValue instanceof Map && schemaType.equals(Type.MAP))
            return unionSchemaPos;
        else if (pValue instanceof List && schemaType.equals(Type.ARRAY))
            return unionSchemaPos;
        else if (pValue instanceof Persistent && schemaType.equals(Type.RECORD))
            return unionSchemaPos;
        unionSchemaPos++;
    }
    // default
    return DEFAULT_UNION_SCHEMA;
}

Example 17 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class SchemaHelper method unwindUnionSchema.

/**
   * @return the field's schema; if the field is a union, the schema for the first non-null type from the
   *         union is returned.
   */
public static Schema unwindUnionSchema(Field field) {
    Schema schema = field.schema();
    Type fieldType = schema.getType();
    // If this is a union, check the child types and return the first non-null schema
    if (fieldType == Type.UNION) {
        List<Schema> unionTypes = schema.getTypes();
        for (Schema unionSubSchema : unionTypes) {
            if (unionSubSchema.getType() != Type.NULL) {
                return unionSubSchema;
            }
        }
    }
    return schema;
}

Example 18 with Type

use of org.apache.avro.Schema.Type in project pinot by linkedin.

the class AvroQueryGenerator method isSingleValueField.

private static boolean isSingleValueField(Field field) {
    org.apache.avro.Schema fieldSchema = field.schema();
    fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);
    final Type type = fieldSchema.getType();
    if (type == Type.ARRAY) {
        return false;
    }
    return true;
}

Example 19 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class BootstrapAuditMain method main.

/**
   * @param args
   */
public static void main(String[] args) throws Exception {
    BootstrapSeederMain.init(args);
    BootstrapSeederMain.StaticConfig staticConfig = BootstrapSeederMain.getStaticConfig();
    int interval = staticConfig.getController().getCommitInterval();
    int sourceChunkSize = staticConfig.getController().getNumRowsPerQuery();
    List<OracleTriggerMonitoredSourceInfo> sources = BootstrapSeederMain.getSources();
    BootstrapDBSeeder seeder = BootstrapSeederMain.getSeeder();
    BootstrapSrcDBEventReader seedController = BootstrapSeederMain.getReader();
    Map<String, String> pKeyNameMap = seedController.getpKeyNameMap();
    Map<String, DbusEventKey.KeyType> pKeyTypeMap = seedController.getpKeyTypeMap();
    for (OracleTriggerMonitoredSourceInfo source : sources) {
        short srcId = source.getSourceId();
        new ConcurrentHashMap<Long, ResultSetEntry>();
        OracleTableReader oracleReader = null;
        MySQLTableReader mySQLReader = null;
        try {
            SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(staticConfig.getSchemaRegistry().getFileSystem());
            Map<Short, String> schemaSet = schemaRegistry.fetchAllSchemaVersionsBySourceName(source.getSourceName());
            VersionedSchemaSet vSchemaSet = new VersionedSchemaSet();
            Iterator<Map.Entry<Short, String>> it = schemaSet.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Short, String> pairs = it.next();
                Schema s = Schema.parse(pairs.getValue());
                VersionedSchema vs = new VersionedSchema(s.getFullName(), pairs.getKey(), s, null);
                vSchemaSet.add(vs);
            }
            /* Try and identify the schema key */
            VersionedSchema vschema = schemaRegistry.fetchLatestVersionedSchemaBySourceName(source.getSourceName());
            Schema schema = Schema.parse(vschema.getSchema().toString());
            LOG.info("Schema =" + vschema.getSchema() + "version=" + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());
            /* Determine type of field txn */
            Field txnFieldType = schema.getField("txn");
            if (txnFieldType == null) {
                throw new Exception("Unable to find field called 'txn'. Cannot proceeed\n");
            }
            Type txnType = SchemaHelper.getAnyType(txnFieldType);
            /*
         * Determine primary key of schema. This is assumed to be invariant
         * across versions
         */
            String keyOverrideName = SchemaHelper.getMetaField(schema, "pk");
            String keyColumnName = "key";
            if (null != keyOverrideName) {
                keyColumnName = keyOverrideName;
            }
            Field pkeyField = schema.getField(keyColumnName);
            if (null == pkeyField) {
                keyColumnName = "id";
                pkeyField = schema.getField("id");
            }
            if (null == pkeyField) {
                throw new Exception("Unable to get the primary key for schema. Schema is :" + schema);
            }
            DbusEventAvroDecoder decoder = new DbusEventAvroDecoder(vSchemaSet);
            BootstrapAuditTester auditor = new BootstrapAuditTester(schema, BootstrapSrcDBEventReader.getTableName(source));
            List<BootstrapAuditTester> auditors = new ArrayList<BootstrapAuditTester>();
            auditors.add(auditor);
            oracleReader = new OracleTableReader(BootstrapSeederMain.getDataStore().getConnection(), BootstrapSrcDBEventReader.getTableName(source), pkeyField, SchemaHelper.getMetaField(pkeyField, "dbFieldName"), SchemaHelper.getAnyType(pkeyField), sourceChunkSize, seedController.getPKIndex(source), seedController.getQueryHint(source));
            mySQLReader = new MySQLTableReader(seeder.getConnection(), // THis is the primary
            seeder.getTableName(srcId), // THis is the primary
            pkeyField, // THis is the primary
            "id", // bootstrapDB
            SchemaHelper.getAnyType(pkeyField), interval);
            double samplePct = BootstrapSeederMain.getValidationSamplePct();
            TableComparator comparator = new TableComparator(oracleReader, mySQLReader, auditor, decoder, interval, pKeyNameMap.get(source.getEventView()), pKeyTypeMap.get(source.getEventView()), txnType, samplePct);
            boolean success = false;
            if (BootstrapSeederMain.getValidationType().equals("point")) {
                success = comparator.compareRecordsPoint();
            } else if (BootstrapSeederMain.getValidationType().equals("pointBs")) {
                success = comparator.compareRecordsPointBs();
            } else {
                success = comparator.compareRecordsNew();
            }
            if (success)
                LOG.info("Audit completed successfully");
            else
                LOG.error("Audit FAILED !!! ");
        } catch (Exception ex) {
            LOG.error("Caught an exception ex", ex);
            throw ex;
        } finally {
            if (null != oracleReader)
                oracleReader.close();
        }
    }
    DBHelper.close(seeder.getConnection());
}

Example 20 with Type

use of org.apache.avro.Schema.Type in project databus by linkedin.

the class BootstrapAuditTester method compareField.

private boolean compareField(Field f, Object databaseFieldValue, Object avroField) {
    // NULL condition handled
    if (databaseFieldValue == avroField) {
        return true;
    }
    if (databaseFieldValue == null) {
        // avroField cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " null databaseFieldValue but non-null avroField ");
        return false;
    }
    if (avroField == null) {
        // databaseFieldValue cannot also be null or first conditional would have triggered
        LOG.error("compareField error: " + " field=" + f.name() + " non-null databaseFieldValue but null avroField ");
        return false;
    }
    try {
        // == f.schema() if f is not a union
        Schema fieldSchema = SchemaHelper.unwindUnionSchema(f);
        Type avroFieldType = fieldSchema.getType();
        if (_sDebug) {
            LOG.debug("Checking for type:" + avroFieldType + ", Field:" + f.name() + ", Exp:" + databaseFieldValue + ", Got:" + avroField);
        }
        switch(avroFieldType) {
            case BOOLEAN:
                assertEquals(f.name(), databaseFieldValue, avroField);
                break;
            case BYTES:
                byte[] byteArr = null;
                if (databaseFieldValue instanceof Blob) {
                    Blob b = (Blob) databaseFieldValue;
                    byteArr = b.getBytes(1, (int) b.length());
                } else {
                    byteArr = (byte[]) databaseFieldValue;
                }
                assertEquals(f.name(), byteArr, avroField);
                break;
            case DOUBLE:
                assertEquals(f.name(), new Double(((Number) databaseFieldValue).doubleValue()), (avroField));
                break;
            case FLOAT:
                assertEquals(f.name(), new Float(((Number) databaseFieldValue).floatValue()), (avroField));
                break;
            case INT:
                assertEquals(f.name(), Integer.valueOf(((Number) databaseFieldValue).intValue()), (avroField));
                break;
            case LONG:
                if (databaseFieldValue instanceof Number) {
                    long lvalue = ((Number) databaseFieldValue).longValue();
                    assertEquals(f.name(), lvalue, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Timestamp) {
                    long time = ((Timestamp) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else if (databaseFieldValue instanceof Date) {
                    long time = ((Date) databaseFieldValue).getTime();
                    assertEquals(f.name(), time, ((Long) avroField).longValue());
                } else {
                    Class timestampClass = null, dateClass = null;
                    try {
                        timestampClass = OracleJarUtils.loadClass("oracle.sql.TIMESTAMP");
                        dateClass = OracleJarUtils.loadClass("oracle.sql.DATE");
                    } catch (Exception e) {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long. Unable to get Oracle datatypes " + e.getMessage();
                        LOG.error(errMsg);
                        throw new EventCreationException(errMsg);
                    }
                    if (timestampClass.isInstance(databaseFieldValue)) {
                        try {
                            Object tsc = timestampClass.cast(databaseFieldValue);
                            Method dateValueMethod = timestampClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(tsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.TIMESTAMP value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else if (dateClass.isInstance(databaseFieldValue)) {
                        try {
                            Object dsc = dateClass.cast(databaseFieldValue);
                            Method dateValueMethod = dateClass.getMethod("dateValue");
                            Date dateValue = (Date) dateValueMethod.invoke(dsc);
                            long time = dateValue.getTime();
                            assertEquals(f.name(), time, ((Long) avroField).longValue());
                        } catch (Exception ex) {
                            String errMsg = "SQLException reading oracle.sql.DATE value for field " + f.name();
                            LOG.error(errMsg);
                            throw new RuntimeException(errMsg, ex);
                        }
                    } else {
                        String errMsg = "Cannot convert " + databaseFieldValue.getClass() + " to long for field " + f.name();
                        LOG.error(errMsg);
                        throw new RuntimeException();
                    }
                }
                break;
            case STRING:
                if (databaseFieldValue instanceof Clob) {
                    String text = null;
                    try {
                        text = OracleAvroGenericEventFactory.extractClobText((Clob) databaseFieldValue, f.name());
                    } catch (EventCreationException ex) {
                        LOG.error("compareField error: " + ex.getMessage(), ex);
                    }
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                } else {
                    String text = databaseFieldValue.toString();
                    assertEquals(f.name(), text, ((Utf8) avroField).toString());
                }
                break;
            case NULL:
                assertNull(f.name(), databaseFieldValue);
                assertNull(f.name(), avroField);
                break;
            case ARRAY:
                GenericArray<GenericRecord> avroArray = (GenericArray<GenericRecord>) avroField;
                Schema elementSchema = fieldSchema.getElementType();
                Array array = (Array) databaseFieldValue;
                ResultSet arrayResultSet = array.getResultSet();
                int i = 0;
                while (arrayResultSet.next()) {
                    // Get the underlying structure from the database. Oracle returns the structure in the
                    // second column of the array's ResultSet
                    Struct struct = (Struct) arrayResultSet.getObject(2);
                    Object[] attributes = struct.getAttributes();
                    GenericRecord avroElement = avroArray.get(i++);
                    // have to use dbFieldPosition recorded in the schema definition.
                    for (Field field : elementSchema.getFields()) {
                        int dbFieldPosition = Integer.valueOf(SchemaHelper.getMetaField(field, "dbFieldPosition"));
                        Object dbFieldValue = attributes[dbFieldPosition];
                        Object avroFieldValue = avroElement.get(field.name());
                        compareField(field, dbFieldValue, avroFieldValue);
                    }
                }
                break;
            case RECORD:
                assert (compareRecord(fieldSchema, (Struct) databaseFieldValue, (GenericRecord) avroField)) : "comparison of Avro 'record' type failed";
                break;
            case ENUM:
            case FIXED:
            case MAP:
            case UNION:
            default:
                String msg = "Audit for these fields not yet implemented for: " + fieldSchema.getName() + ", Avro type: " + avroFieldType;
                LOG.error(msg);
                throw new RuntimeException(msg);
        }
    } catch (AssertionError err) {
        LOG.error("compareField error: " + err.getMessage() + " field= " + f.name());
        return false;
    } catch (ClassCastException ce) {
        LOG.error("compareField error: " + ce.getMessage() + " field=" + f.name(), ce);
        return false;
    } catch (Exception ex) {
        LOG.error("compareField error: " + ex.getMessage() + " field=" + f.name(), ex);
        return false;
    }
    return true;
}