Examples with JsonObjectExt com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt used on opensource projects

Example 1 with JsonObjectExt

use of com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt in project avro-util by linkedin.

the class AvscParser method parseNamedSchema.

private AvroNamedSchema parseNamedSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer extraProps) {
    Located<String> nameStr = getRequiredString(objectNode, "name", () -> avroType + " is a named type");
    Located<String> namespaceStr = getOptionalString(objectNode, "namespace");
    // technically the avro spec does not allow "doc" on type fixed, but screw that
    Located<String> docStr = getOptionalString(objectNode, "doc");
    String name = nameStr.getValue();
    String namespace = namespaceStr != null ? namespaceStr.getValue() : null;
    String doc = docStr != null ? docStr.getValue() : null;
    String schemaSimpleName;
    String schemaNamespace;
    if (name.contains(".")) {
        // the name specified is a full name (namespace included)
        context.addIssue(AvscIssues.useOfFullName(new CodeLocation(context.getUri(), nameStr.getLocation(), nameStr.getLocation()), avroType, name));
        if (namespace != null) {
            // namespace will be ignored, but it's confusing to even list it
            context.addIssue(AvscIssues.ignoredNamespace(new CodeLocation(context.getUri(), namespaceStr.getLocation(), namespaceStr.getLocation()), avroType, namespace, name));
        }
        // TODO - validate names (no ending in dot, no spaces, etc)
        int lastDot = name.lastIndexOf('.');
        schemaSimpleName = name.substring(lastDot + 1);
        schemaNamespace = name.substring(0, lastDot);
    } else {
        schemaSimpleName = name;
        schemaNamespace = namespace;
    }
    // != null
    String contextNamespace = context.getCurrentNamespace();
    boolean namespaceChanged = false;
    // check if context namespace changed
    if (schemaNamespace != null) {
        if (!contextNamespace.equals(schemaNamespace)) {
            context.pushNamespace(schemaNamespace);
            namespaceChanged = true;
            contextNamespace = schemaNamespace;
        }
    }
    AvroNamedSchema namedSchema;
    switch(avroType) {
        case RECORD:
            AvroRecordSchema recordSchema = new AvroRecordSchema(codeLocation, schemaSimpleName, contextNamespace, doc, extraProps);
            JsonArrayExt fieldsNode = getRequiredArray(objectNode, "fields", () -> "all avro records must have fields");
            List<AvroSchemaField> fields = new ArrayList<>(fieldsNode.size());
            for (int fieldNum = 0; fieldNum < fieldsNode.size(); fieldNum++) {
                // !=null
                JsonValueExt fieldDeclNode = (JsonValueExt) fieldsNode.get(fieldNum);
                JsonValue.ValueType fieldNodeType = fieldDeclNode.getValueType();
                if (fieldNodeType != JsonValue.ValueType.OBJECT) {
                    throw new AvroSyntaxException("field " + fieldNum + " for record " + schemaSimpleName + " at " + fieldDeclNode.getStartLocation() + " expected to be an OBJECT, not a " + JsonPUtil.describe(fieldNodeType) + " (" + fieldDeclNode + ")");
                }
                TextLocation fieldStartLocation = Util.convertLocation(fieldDeclNode.getStartLocation());
                TextLocation fieldEndLocation = Util.convertLocation(fieldDeclNode.getEndLocation());
                CodeLocation fieldCodeLocation = new CodeLocation(context.getUri(), fieldStartLocation, fieldEndLocation);
                JsonObjectExt fieldDecl = (JsonObjectExt) fieldDeclNode;
                Located<String> fieldName = getRequiredString(fieldDecl, "name", () -> "all record fields must have a name");
                JsonValueExt fieldTypeNode = getRequiredNode(fieldDecl, "type", () -> "all record fields must have a type");
                SchemaOrRef fieldSchema = parseSchemaDeclOrRef(fieldTypeNode, context, false);
                JsonValueExt fieldDefaultValueNode = fieldDecl.get("default");
                AvroLiteral defaultValue = null;
                if (fieldDefaultValueNode != null) {
                    if (fieldSchema.isResolved()) {
                        LiteralOrIssue defaultValurOrIssue = parseLiteral(fieldDefaultValueNode, fieldSchema.getSchema(), fieldName.getValue(), context);
                        if (defaultValurOrIssue.getIssue() == null) {
                            defaultValue = defaultValurOrIssue.getLiteral();
                        }
                    // TODO - handle issues
                    } else {
                        // TODO - implement delayed default value parsing
                        throw new UnsupportedOperationException("delayed parsing of default value for " + fieldName.getValue() + " TBD");
                    }
                }
                LinkedHashMap<String, JsonValueExt> props = parseExtraProps(fieldDecl, CORE_FIELD_PROPERTIES);
                JsonPropertiesContainer propsContainer = props.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(props);
                AvroSchemaField field = new AvroSchemaField(fieldCodeLocation, fieldName.getValue(), null, fieldSchema, defaultValue, propsContainer);
                fields.add(field);
            }
            recordSchema.setFields(fields);
            namedSchema = recordSchema;
            break;
        case ENUM:
            JsonArrayExt symbolsNode = getRequiredArray(objectNode, "symbols", () -> "all avro enums must have symbols");
            List<String> symbols = new ArrayList<>(symbolsNode.size());
            for (int ordinal = 0; ordinal < symbolsNode.size(); ordinal++) {
                JsonValueExt symbolNode = (JsonValueExt) symbolsNode.get(ordinal);
                JsonValue.ValueType symbolNodeType = symbolNode.getValueType();
                if (symbolNodeType != JsonValue.ValueType.STRING) {
                    throw new AvroSyntaxException("symbol " + ordinal + " for enum " + schemaSimpleName + " at " + symbolNode.getStartLocation() + " expected to be a STRING, not a " + JsonPUtil.describe(symbolNodeType) + " (" + symbolNode + ")");
                }
                symbols.add(symbolNode.toString());
            }
            String defaultSymbol = null;
            Located<String> defaultStr = getOptionalString(objectNode, "default");
            if (defaultStr != null) {
                defaultSymbol = defaultStr.getValue();
                if (!symbols.contains(defaultSymbol)) {
                    context.addIssue(AvscIssues.badEnumDefaultValue(locationOf(context.getUri(), defaultStr), defaultSymbol, schemaSimpleName, symbols));
                    // TODO - support "fixing" by selecting 1st symbol as default?
                    defaultSymbol = null;
                }
            }
            namedSchema = new AvroEnumSchema(codeLocation, schemaSimpleName, contextNamespace, doc, symbols, defaultSymbol, extraProps);
            break;
        case FIXED:
            JsonValueExt sizeNode = getRequiredNode(objectNode, "size", () -> "fixed types must have a size property");
            if (sizeNode.getValueType() != JsonValue.ValueType.NUMBER || !(((JsonNumberExt) sizeNode).isIntegral())) {
                throw new AvroSyntaxException("size for fixed " + schemaSimpleName + " at " + sizeNode.getStartLocation() + " expected to be an INTEGER, not a " + JsonPUtil.describe(sizeNode.getValueType()) + " (" + sizeNode + ")");
            }
            int fixedSize = ((JsonNumberExt) sizeNode).intValue();
            Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(objectNode, context, avroType, codeLocation);
            if (logicalTypeResult.hasIssues()) {
                context.addIssues(logicalTypeResult.getIssues());
            }
            namedSchema = new AvroFixedSchema(codeLocation, schemaSimpleName, contextNamespace, doc, fixedSize, logicalTypeResult.getData(), extraProps);
            break;
        default:
            throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
    }
    if (namespaceChanged) {
        context.popNamespace();
    }
    return namedSchema;
}

Example 2 with JsonObjectExt

use of com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt in project avro-util by linkedin.

the class AvscParser method parseLogicalType.

private Parsed<AvroLogicalType> parseLogicalType(// type declaration node
JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation) {
    Parsed<AvroLogicalType> result;
    Located<String> logicalTypeStr = getOptionalString(objectNode, "logicalType");
    AvroLogicalType logicalType;
    if (logicalTypeStr != null) {
        logicalType = AvroLogicalType.fromJson(logicalTypeStr.getValue());
        if (logicalType == null) {
            return new Parsed<>(AvscIssues.unknownLogicalType(locationOf(context.getUri(), logicalTypeStr), logicalTypeStr.getValue()));
        } else {
            // we have a logicalType, see if it matches the type
            if (!logicalType.getParentTypes().contains(avroType)) {
                return new Parsed<>(AvscIssues.mismatchedLogicalType(locationOf(context.getUri(), logicalTypeStr), logicalType, avroType));
            }
            return new Parsed<>(logicalType);
        }
    } else {
        // no logical type (also no issues)
        return new Parsed<>((AvroLogicalType) null);
    }
}

Example 3 with JsonObjectExt

use of com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt in project avro-util by linkedin.

the class AvscParser method parseComplexSchema.

private SchemaOrRef parseComplexSchema(JsonObjectExt objectNode, AvscFileParseContext context, boolean topLevel) {
    CodeLocation codeLocation = locationOf(context.getUri(), objectNode);
    Located<String> typeStr = getRequiredString(objectNode, "type", () -> "it is a schema declaration");
    AvroType avroType = AvroType.fromJson(typeStr.getValue());
    if (avroType == null) {
        throw new AvroSyntaxException("unknown avro type \"" + typeStr.getValue() + "\" at " + typeStr.getLocation() + ". expecting \"record\", \"enum\" or \"fixed\"");
    }
    LinkedHashMap<String, JsonValueExt> propsMap = parseExtraProps(objectNode, CORE_SCHEMA_PROPERTIES);
    JsonPropertiesContainer props = propsMap.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(propsMap);
    AvroSchema definedSchema;
    if (avroType.isNamed()) {
        definedSchema = parseNamedSchema(objectNode, context, avroType, codeLocation, props);
    } else if (avroType.isCollection()) {
        definedSchema = parseCollectionSchema(objectNode, context, avroType, codeLocation, props);
    } else if (avroType.isPrimitive()) {
        definedSchema = parseDecoratedPrimitiveSchema(objectNode, context, avroType, codeLocation, props);
    } else {
        throw new IllegalStateException("unhandled avro type " + avroType + " at " + typeStr.getLocation());
    }
    context.defineSchema(definedSchema, topLevel);
    return new SchemaOrRef(codeLocation, definedSchema);
}

Example 4 with JsonObjectExt

use of com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt in project avro-util by linkedin.

the class AvscParser method parse.

public AvscParseResult parse(String avsc) {
    JsonReaderExt jsonReader = new JsonReaderWithLocations(new StringReader(avsc), null);
    JsonObjectExt root;
    AvscFileParseContext context = new AvscFileParseContext(avsc);
    AvscParseResult result = new AvscParseResult();
    try {
        root = jsonReader.readObject();
    } catch (JsonParsingException e) {
        Throwable rootCause = Util.rootCause(e);
        String message = rootCause.getMessage();
        if (message != null && message.startsWith("Unexpected char 47")) {
            // 47 is ascii for '/' (utf-8 matches ascii on "low" characters). we are going to assume this means
            // someone tried putting a //comment into an avsc source
            result.recordError(new JsonParseException("comments not supported in json at" + e.getLocation(), e));
        } else {
            result.recordError(new JsonParseException("json parse error at " + e.getLocation(), e));
        }
        return result;
    } catch (Exception e) {
        result.recordError(new JsonParseException("unknown json parse error", e));
        return result;
    }
    try {
        parseSchemaDeclOrRef(root, context, true);
        context.resolveReferences();
        result.recordParseComplete(context);
    } catch (Exception parseIssue) {
        result.recordError(parseIssue);
    }
    return result;
}

Example 5 with JsonObjectExt

use of com.linkedin.avroutil1.parser.jsonpext.JsonObjectExt in project avro-util by linkedin.

the class AvscParser method parseDecoratedPrimitiveSchema.

private AvroPrimitiveSchema parseDecoratedPrimitiveSchema(JsonObjectExt primitiveNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer props) {
    AvroLogicalType logicalType = null;
    int scale = 0;
    int precision = 0;
    Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(primitiveNode, context, avroType, codeLocation);
    if (logicalTypeResult.hasIssues()) {
        context.addIssues(logicalTypeResult.getIssues());
    }
    // might be null
    logicalType = logicalTypeResult.getData();
    Parsed<AvroJavaStringRepresentation> stringRepResult = parseStringRepresentation(primitiveNode, context, avroType, codeLocation);
    AvroJavaStringRepresentation stringRep = null;
    if (stringRepResult.hasIssues()) {
        context.addIssues(stringRepResult.getIssues());
    }
    if (AvroType.STRING.equals(avroType) && stringRepResult.hasData()) {
        stringRep = stringRepResult.getData();
    }
    Located<Integer> precisionResult = getOptionalInteger(primitiveNode, "precision", context);
    Located<Integer> scaleResult = getOptionalInteger(primitiveNode, "scale", context);
    boolean scaleAndPrecisionExpected = AvroType.BYTES.equals(avroType) && AvroLogicalType.DECIMAL.equals(logicalType);
    if (scaleAndPrecisionExpected) {
        boolean scaleAndPrecisionOK = true;
        int precisionValue = 0;
        int scaleValue = 0;
        // precision is actually required
        if (precisionResult == null) {
            context.addIssue(AvscIssues.precisionRequiredAndNotSet(codeLocation, avroType, logicalType));
            scaleAndPrecisionOK = false;
        }
        if (scaleAndPrecisionOK && scaleResult != null) {
            precisionValue = precisionResult.getValue();
            scaleValue = scaleResult.getValue();
            if (precisionValue < scaleValue) {
                context.addIssue(AvscIssues.precisionSmallerThanScale(locationOf(context.getUri(), precisionResult), precisionValue, locationOf(context.getUri(), scaleResult), scaleValue));
                scaleAndPrecisionOK = false;
            }
        }
        if (scaleAndPrecisionOK) {
            scale = scaleValue;
            precision = precisionValue;
        } else {
            // "cancel" the logicalType
            logicalType = null;
        }
    }
    return new AvroPrimitiveSchema(codeLocation, avroType, logicalType, stringRep, scale, precision, props);
}