Examples with AvroSchema com.linkedin.avroutil1.model.AvroSchema used on opensource projects

Example 11 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscSchemaWriter method writeSchema.

protected JsonValue writeSchema(AvroSchema schema, AvscWriterContext context, AvscWriterConfig config) {
    AvroType type = schema.type();
    JsonObjectBuilder definitionBuilder;
    switch(type) {
        case ENUM:
        case FIXED:
        case RECORD:
            return writeNamedSchema((AvroNamedSchema) schema, context, config);
        case ARRAY:
            AvroArraySchema arraySchema = (AvroArraySchema) schema;
            definitionBuilder = Json.createObjectBuilder();
            definitionBuilder.add("type", "array");
            definitionBuilder.add("items", writeSchema(arraySchema.getValueSchema(), context, config));
            emitJsonProperties(schema, context, config, definitionBuilder);
            return definitionBuilder.build();
        case MAP:
            AvroMapSchema mapSchema = (AvroMapSchema) schema;
            definitionBuilder = Json.createObjectBuilder();
            definitionBuilder.add("type", "map");
            definitionBuilder.add("values", writeSchema(mapSchema.getValueSchema(), context, config));
            emitJsonProperties(schema, context, config, definitionBuilder);
            return definitionBuilder.build();
        case UNION:
            AvroUnionSchema unionSchema = (AvroUnionSchema) schema;
            JsonArrayBuilder unionBuilder = Json.createArrayBuilder();
            for (SchemaOrRef unionBranch : unionSchema.getTypes()) {
                // will throw if unresolved ref
                AvroSchema branchSchema = unionBranch.getSchema();
                unionBuilder.add(writeSchema(branchSchema, context, config));
            }
            return unionBuilder.build();
        default:
            AvroPrimitiveSchema primitiveSchema = (AvroPrimitiveSchema) schema;
            if (!primitiveSchema.hasProperties()) {
                return Json.createValue(primitiveSchema.type().name().toLowerCase(Locale.ROOT));
            }
            definitionBuilder = Json.createObjectBuilder();
            definitionBuilder.add("type", primitiveSchema.type().toTypeName());
            emitJsonProperties(primitiveSchema, context, config, definitionBuilder);
            return definitionBuilder.build();
    }
}

Example 12 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscFileParseContext method defineSchema.

/**
 * defines a new (completely parsed) schema
 * @param schema new schema
 * @param isTopLevel true if schema is the top-level schema of the avsc being parsed
 */
public void defineSchema(AvroSchema schema, boolean isTopLevel) {
    if (schema == null) {
        throw new IllegalArgumentException("schema cannot be null");
    }
    if (isTopLevel) {
        if (topLevelSchema != null) {
            throw new IllegalStateException("cannot set " + schema + " as top level schema " + "for context as thats already set to " + topLevelSchema);
        }
        topLevelSchema = schema;
    }
    definedSchemas.add(schema);
    if (schema.type().isNamed()) {
        AvroNamedSchema namedSchema = (AvroNamedSchema) schema;
        AvroSchema other = definedNamedSchemas.putIfAbsent(namedSchema.getFullName(), schema);
        if (other != null) {
            throw new AvroSyntaxException(schema + " defined in " + schema.getCodeLocation() + " conflicts with " + other);
        }
    }
}

Example 13 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscFileParseContext method resolveReferences.

private void resolveReferences(AvroSchema schema) {
    AvroType type = schema.type();
    switch(type) {
        case RECORD:
            AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
            List<AvroSchemaField> fields = recordSchema.getFields();
            for (AvroSchemaField field : fields) {
                SchemaOrRef fieldSchema = field.getSchemaOrRef();
                resolveReferences(fieldSchema);
            }
            break;
        case UNION:
            AvroUnionSchema unionSchema = (AvroUnionSchema) schema;
            List<SchemaOrRef> types = unionSchema.getTypes();
            for (SchemaOrRef unionType : types) {
                resolveReferences(unionType);
            }
            break;
        case ARRAY:
            AvroArraySchema arraySchema = (AvroArraySchema) schema;
            SchemaOrRef arrayValuesType = arraySchema.getValueSchemaOrRef();
            resolveReferences(arrayValuesType);
            break;
        case MAP:
            AvroMapSchema mapSchema = (AvroMapSchema) schema;
            SchemaOrRef mapValuesType = mapSchema.getValueSchemaOrRef();
            resolveReferences(mapValuesType);
            break;
        default:
            break;
    }
}

Example 14 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscFileParseContext method resolveReferences.

private void resolveReferences(SchemaOrRef possiblyRef) {
    if (possiblyRef.isResolved()) {
        // either an already- resolved reference or an inline definition
        if (possiblyRef.getDecl() != null) {
            // recurse into inline definitions
            resolveReferences(possiblyRef.getDecl());
        }
    } else {
        // unresolved (and so must be a) reference
        String simpleName = possiblyRef.getRef();
        AvroSchema simpleNameResolution = definedNamedSchemas.get(simpleName);
        AvroSchema inheritedNameResolution = null;
        String inheritedName = possiblyRef.getInheritedName();
        if (inheritedName != null) {
            inheritedNameResolution = definedNamedSchemas.get(inheritedName);
        }
        if (inheritedNameResolution != null) {
            possiblyRef.setResolvedTo(inheritedNameResolution);
            if (simpleNameResolution != null) {
                String msg = "Two different schemas found for reference " + simpleName + " with inherited name " + inheritedName + ". Only one should exist.";
                AvscIssue issue = new AvscIssue(possiblyRef.getCodeLocation(), IssueSeverity.WARNING, msg, new IllegalStateException(msg));
                issues.add(issue);
            }
        } else if (simpleNameResolution != null) {
            possiblyRef.setResolvedTo(simpleNameResolution);
        } else {
            externalReferences.add(possiblyRef);
        }
    }
}

Example 15 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscParser method parseLiteral.

private LiteralOrIssue parseLiteral(JsonValueExt literalNode, AvroSchema schema, String fieldName, AvscFileParseContext context) {
    AvroType avroType = schema.type();
    JsonValue.ValueType jsonType = literalNode.getValueType();
    AvscIssue issue;
    BigInteger bigIntegerValue;
    BigDecimal bigDecimalValue;
    byte[] bytes;
    switch(avroType) {
        case NULL:
            if (jsonType != JsonValue.ValueType.NULL) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroNullLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode)));
        case BOOLEAN:
            if (jsonType != JsonValue.ValueType.FALSE && jsonType != JsonValue.ValueType.TRUE) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            boolean boolValue = jsonType == JsonValue.ValueType.TRUE;
            return new LiteralOrIssue(new AvroBooleanLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), boolValue));
        case INT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt intNode = (JsonNumberExt) literalNode;
            if (!intNode.isIntegral()) {
                // TODO - be more specific about this error (int vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = intNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_INT) > 0 || bigIntegerValue.compareTo(MIN_INT) < 0) {
                // TODO - be more specific about this error (out of signed int range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroIntegerLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.intValueExact()));
        case LONG:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt longNode = (JsonNumberExt) literalNode;
            if (!longNode.isIntegral()) {
                // TODO - be more specific about this error (long vs float)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            bigIntegerValue = longNode.bigIntegerValue();
            if (bigIntegerValue.compareTo(MAX_LONG) > 0 || bigIntegerValue.compareTo(MIN_LONG) < 0) {
                // TODO - be more specific about this error (out of signed long range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroLongLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.longValueExact()));
        case FLOAT:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt floatNode = (JsonNumberExt) literalNode;
            bigDecimalValue = floatNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_FLOAT) > 0) {
                // TODO - be more specific about this error (out of float range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFloatLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.floatValue()));
        case DOUBLE:
            if (jsonType != JsonValue.ValueType.NUMBER) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonNumberExt doubleNode = (JsonNumberExt) literalNode;
            bigDecimalValue = doubleNode.bigDecimalValue();
            if (bigDecimalValue.compareTo(MAX_DOUBLE) > 0) {
                // TODO - be more specific about this error (out of double range)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroDoubleLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.doubleValue()));
        case BYTES:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt bytesNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = bytesNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            return new LiteralOrIssue(new AvroBytesLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bytes));
        case FIXED:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
            JsonStringExt fixedNode = (JsonStringExt) literalNode;
            // spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
            bytes = fixedNode.getString().getBytes(StandardCharsets.ISO_8859_1);
            if (bytes.length != fixedSchema.getSize()) {
                // TODO - be more specific about this error (wrong length)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroFixedLiteral(fixedSchema, locationOf(context.getUri(), literalNode), bytes));
        case STRING:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            JsonStringExt stringNode = (JsonStringExt) literalNode;
            return new LiteralOrIssue(new AvroStringLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), stringNode.getString()));
        case ENUM:
            if (jsonType != JsonValue.ValueType.STRING) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
            JsonStringExt enumNode = (JsonStringExt) literalNode;
            String enumValue = enumNode.getString();
            if (!enumSchema.getSymbols().contains(enumValue)) {
                // TODO - be more specific about this error (unknown symbol)
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            return new LiteralOrIssue(new AvroEnumLiteral(enumSchema, locationOf(context.getUri(), literalNode), enumValue));
        case ARRAY:
            if (jsonType != JsonValue.ValueType.ARRAY) {
                issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                context.addIssue(issue);
                return new LiteralOrIssue(issue);
            }
            AvroArraySchema arraySchema = (AvroArraySchema) schema;
            AvroSchema valueSchema = arraySchema.getValueSchema();
            JsonArrayExt arrayNode = (JsonArrayExt) literalNode;
            ArrayList<AvroLiteral> values = new ArrayList<>(arrayNode.size());
            for (int i = 0; i < arrayNode.size(); i++) {
                JsonValueExt valueNode = (JsonValueExt) arrayNode.get(i);
                LiteralOrIssue value = parseLiteral(valueNode, valueSchema, fieldName, context);
                // issues parsing any member value mean a failure to parse the array as a whole
                if (value.getIssue() != null) {
                    // TODO - be more specific about this error (unparsable array element i)
                    // TODO - add "causedBy" to AvscIssue and use it here
                    issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
                    context.addIssue(issue);
                    return new LiteralOrIssue(issue);
                }
                values.add(value.getLiteral());
            }
            return new LiteralOrIssue(new AvroArrayLiteral(arraySchema, locationOf(context.getUri(), literalNode), values));
        default:
            throw new UnsupportedOperationException("dont know how to parse a " + avroType + " at " + literalNode.getStartLocation() + " out of a " + literalNode.getValueType() + " (" + literalNode + ")");
    }
}