use of com.linkedin.avroutil1.parser.jsonpext.JsonValueExt in project avro-util by linkedin.
the class AvscParser method parseNamedSchema.
private AvroNamedSchema parseNamedSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer extraProps) {
Located<String> nameStr = getRequiredString(objectNode, "name", () -> avroType + " is a named type");
Located<String> namespaceStr = getOptionalString(objectNode, "namespace");
// technically the avro spec does not allow "doc" on type fixed, but screw that
Located<String> docStr = getOptionalString(objectNode, "doc");
String name = nameStr.getValue();
String namespace = namespaceStr != null ? namespaceStr.getValue() : null;
String doc = docStr != null ? docStr.getValue() : null;
String schemaSimpleName;
String schemaNamespace;
if (name.contains(".")) {
// the name specified is a full name (namespace included)
context.addIssue(AvscIssues.useOfFullName(new CodeLocation(context.getUri(), nameStr.getLocation(), nameStr.getLocation()), avroType, name));
if (namespace != null) {
// namespace will be ignored, but it's confusing to even list it
context.addIssue(AvscIssues.ignoredNamespace(new CodeLocation(context.getUri(), namespaceStr.getLocation(), namespaceStr.getLocation()), avroType, namespace, name));
}
// TODO - validate names (no ending in dot, no spaces, etc)
int lastDot = name.lastIndexOf('.');
schemaSimpleName = name.substring(lastDot + 1);
schemaNamespace = name.substring(0, lastDot);
} else {
schemaSimpleName = name;
schemaNamespace = namespace;
}
// != null
String contextNamespace = context.getCurrentNamespace();
boolean namespaceChanged = false;
// check if context namespace changed
if (schemaNamespace != null) {
if (!contextNamespace.equals(schemaNamespace)) {
context.pushNamespace(schemaNamespace);
namespaceChanged = true;
contextNamespace = schemaNamespace;
}
}
AvroNamedSchema namedSchema;
switch(avroType) {
case RECORD:
AvroRecordSchema recordSchema = new AvroRecordSchema(codeLocation, schemaSimpleName, contextNamespace, doc, extraProps);
JsonArrayExt fieldsNode = getRequiredArray(objectNode, "fields", () -> "all avro records must have fields");
List<AvroSchemaField> fields = new ArrayList<>(fieldsNode.size());
for (int fieldNum = 0; fieldNum < fieldsNode.size(); fieldNum++) {
// !=null
JsonValueExt fieldDeclNode = (JsonValueExt) fieldsNode.get(fieldNum);
JsonValue.ValueType fieldNodeType = fieldDeclNode.getValueType();
if (fieldNodeType != JsonValue.ValueType.OBJECT) {
throw new AvroSyntaxException("field " + fieldNum + " for record " + schemaSimpleName + " at " + fieldDeclNode.getStartLocation() + " expected to be an OBJECT, not a " + JsonPUtil.describe(fieldNodeType) + " (" + fieldDeclNode + ")");
}
TextLocation fieldStartLocation = Util.convertLocation(fieldDeclNode.getStartLocation());
TextLocation fieldEndLocation = Util.convertLocation(fieldDeclNode.getEndLocation());
CodeLocation fieldCodeLocation = new CodeLocation(context.getUri(), fieldStartLocation, fieldEndLocation);
JsonObjectExt fieldDecl = (JsonObjectExt) fieldDeclNode;
Located<String> fieldName = getRequiredString(fieldDecl, "name", () -> "all record fields must have a name");
JsonValueExt fieldTypeNode = getRequiredNode(fieldDecl, "type", () -> "all record fields must have a type");
SchemaOrRef fieldSchema = parseSchemaDeclOrRef(fieldTypeNode, context, false);
JsonValueExt fieldDefaultValueNode = fieldDecl.get("default");
AvroLiteral defaultValue = null;
if (fieldDefaultValueNode != null) {
if (fieldSchema.isResolved()) {
LiteralOrIssue defaultValurOrIssue = parseLiteral(fieldDefaultValueNode, fieldSchema.getSchema(), fieldName.getValue(), context);
if (defaultValurOrIssue.getIssue() == null) {
defaultValue = defaultValurOrIssue.getLiteral();
}
// TODO - handle issues
} else {
// TODO - implement delayed default value parsing
throw new UnsupportedOperationException("delayed parsing of default value for " + fieldName.getValue() + " TBD");
}
}
LinkedHashMap<String, JsonValueExt> props = parseExtraProps(fieldDecl, CORE_FIELD_PROPERTIES);
JsonPropertiesContainer propsContainer = props.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(props);
AvroSchemaField field = new AvroSchemaField(fieldCodeLocation, fieldName.getValue(), null, fieldSchema, defaultValue, propsContainer);
fields.add(field);
}
recordSchema.setFields(fields);
namedSchema = recordSchema;
break;
case ENUM:
JsonArrayExt symbolsNode = getRequiredArray(objectNode, "symbols", () -> "all avro enums must have symbols");
List<String> symbols = new ArrayList<>(symbolsNode.size());
for (int ordinal = 0; ordinal < symbolsNode.size(); ordinal++) {
JsonValueExt symbolNode = (JsonValueExt) symbolsNode.get(ordinal);
JsonValue.ValueType symbolNodeType = symbolNode.getValueType();
if (symbolNodeType != JsonValue.ValueType.STRING) {
throw new AvroSyntaxException("symbol " + ordinal + " for enum " + schemaSimpleName + " at " + symbolNode.getStartLocation() + " expected to be a STRING, not a " + JsonPUtil.describe(symbolNodeType) + " (" + symbolNode + ")");
}
symbols.add(symbolNode.toString());
}
String defaultSymbol = null;
Located<String> defaultStr = getOptionalString(objectNode, "default");
if (defaultStr != null) {
defaultSymbol = defaultStr.getValue();
if (!symbols.contains(defaultSymbol)) {
context.addIssue(AvscIssues.badEnumDefaultValue(locationOf(context.getUri(), defaultStr), defaultSymbol, schemaSimpleName, symbols));
// TODO - support "fixing" by selecting 1st symbol as default?
defaultSymbol = null;
}
}
namedSchema = new AvroEnumSchema(codeLocation, schemaSimpleName, contextNamespace, doc, symbols, defaultSymbol, extraProps);
break;
case FIXED:
JsonValueExt sizeNode = getRequiredNode(objectNode, "size", () -> "fixed types must have a size property");
if (sizeNode.getValueType() != JsonValue.ValueType.NUMBER || !(((JsonNumberExt) sizeNode).isIntegral())) {
throw new AvroSyntaxException("size for fixed " + schemaSimpleName + " at " + sizeNode.getStartLocation() + " expected to be an INTEGER, not a " + JsonPUtil.describe(sizeNode.getValueType()) + " (" + sizeNode + ")");
}
int fixedSize = ((JsonNumberExt) sizeNode).intValue();
Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(objectNode, context, avroType, codeLocation);
if (logicalTypeResult.hasIssues()) {
context.addIssues(logicalTypeResult.getIssues());
}
namedSchema = new AvroFixedSchema(codeLocation, schemaSimpleName, contextNamespace, doc, fixedSize, logicalTypeResult.getData(), extraProps);
break;
default:
throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
}
if (namespaceChanged) {
context.popNamespace();
}
return namedSchema;
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonValueExt in project avro-util by linkedin.
the class AvscParser method parseComplexSchema.
private SchemaOrRef parseComplexSchema(JsonObjectExt objectNode, AvscFileParseContext context, boolean topLevel) {
CodeLocation codeLocation = locationOf(context.getUri(), objectNode);
Located<String> typeStr = getRequiredString(objectNode, "type", () -> "it is a schema declaration");
AvroType avroType = AvroType.fromJson(typeStr.getValue());
if (avroType == null) {
throw new AvroSyntaxException("unknown avro type \"" + typeStr.getValue() + "\" at " + typeStr.getLocation() + ". expecting \"record\", \"enum\" or \"fixed\"");
}
LinkedHashMap<String, JsonValueExt> propsMap = parseExtraProps(objectNode, CORE_SCHEMA_PROPERTIES);
JsonPropertiesContainer props = propsMap.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(propsMap);
AvroSchema definedSchema;
if (avroType.isNamed()) {
definedSchema = parseNamedSchema(objectNode, context, avroType, codeLocation, props);
} else if (avroType.isCollection()) {
definedSchema = parseCollectionSchema(objectNode, context, avroType, codeLocation, props);
} else if (avroType.isPrimitive()) {
definedSchema = parseDecoratedPrimitiveSchema(objectNode, context, avroType, codeLocation, props);
} else {
throw new IllegalStateException("unhandled avro type " + avroType + " at " + typeStr.getLocation());
}
context.defineSchema(definedSchema, topLevel);
return new SchemaOrRef(codeLocation, definedSchema);
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonValueExt in project avro-util by linkedin.
the class AvscParser method parseExtraProps.
private LinkedHashMap<String, JsonValueExt> parseExtraProps(JsonObjectExt field, Set<String> toIgnore) {
LinkedHashMap<String, JsonValueExt> results = new LinkedHashMap<>(3);
for (Map.Entry<String, JsonValue> entry : field.entrySet()) {
// doc says there are in the order they are in json
String propName = entry.getKey();
if (toIgnore.contains(propName)) {
continue;
}
JsonValueExt propValue = (JsonValueExt) entry.getValue();
results.put(propName, propValue);
}
return results;
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonValueExt in project avro-util by linkedin.
the class TestJsonParsing method testSimpleSchema.
@Test
public void testSimpleSchema() throws Exception {
String avsc = TestUtil.load("schemas/TestRecord.avsc");
JsonReader referenceReader = Json.createReader(new StringReader(avsc));
JsonObject referenceObject = referenceReader.readObject();
JsonReaderExt ourReader = new JsonReaderWithLocations(new StringReader(avsc), null);
JsonObjectExt ourObject = ourReader.readObject();
// assert we read everything correctly. our equals() implementations are not
// commutative with regular json-p objects (yet?)
// noinspection SimplifiableAssertion
Assert.assertTrue(referenceObject.equals(ourObject));
// see that we have text locations for json values
JsonValueExt fields = ourObject.get("fields");
JsonLocation startLocation = fields.getStartLocation();
JsonLocation endLocation = fields.getEndLocation();
Assert.assertEquals(startLocation.getLineNumber(), 6);
Assert.assertEquals(endLocation.getLineNumber(), 70);
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonValueExt in project avro-util by linkedin.
the class AvscParser method parseCollectionSchema.
private AvroCollectionSchema parseCollectionSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer props) {
switch(avroType) {
case ARRAY:
JsonValueExt arrayItemsNode = getRequiredNode(objectNode, "items", () -> "array declarations must have an items property");
SchemaOrRef arrayItemSchema = parseSchemaDeclOrRef(arrayItemsNode, context, false);
return new AvroArraySchema(codeLocation, arrayItemSchema, props);
case MAP:
JsonValueExt mapValuesNode = getRequiredNode(objectNode, "values", () -> "map declarations must have a values property");
SchemaOrRef mapValueSchema = parseSchemaDeclOrRef(mapValuesNode, context, false);
return new AvroMapSchema(codeLocation, mapValueSchema, props);
default:
throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
}
}
Aggregations