use of com.linkedin.avroutil1.parser.jsonpext.JsonNumberExt in project avro-util by linkedin.
the class AvscParser method parseNamedSchema.
private AvroNamedSchema parseNamedSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer extraProps) {
Located<String> nameStr = getRequiredString(objectNode, "name", () -> avroType + " is a named type");
Located<String> namespaceStr = getOptionalString(objectNode, "namespace");
// technically the avro spec does not allow "doc" on type fixed, but screw that
Located<String> docStr = getOptionalString(objectNode, "doc");
String name = nameStr.getValue();
String namespace = namespaceStr != null ? namespaceStr.getValue() : null;
String doc = docStr != null ? docStr.getValue() : null;
String schemaSimpleName;
String schemaNamespace;
if (name.contains(".")) {
// the name specified is a full name (namespace included)
context.addIssue(AvscIssues.useOfFullName(new CodeLocation(context.getUri(), nameStr.getLocation(), nameStr.getLocation()), avroType, name));
if (namespace != null) {
// namespace will be ignored, but it's confusing to even list it
context.addIssue(AvscIssues.ignoredNamespace(new CodeLocation(context.getUri(), namespaceStr.getLocation(), namespaceStr.getLocation()), avroType, namespace, name));
}
// TODO - validate names (no ending in dot, no spaces, etc)
int lastDot = name.lastIndexOf('.');
schemaSimpleName = name.substring(lastDot + 1);
schemaNamespace = name.substring(0, lastDot);
} else {
schemaSimpleName = name;
schemaNamespace = namespace;
}
// != null
String contextNamespace = context.getCurrentNamespace();
boolean namespaceChanged = false;
// check if context namespace changed
if (schemaNamespace != null) {
if (!contextNamespace.equals(schemaNamespace)) {
context.pushNamespace(schemaNamespace);
namespaceChanged = true;
contextNamespace = schemaNamespace;
}
}
AvroNamedSchema namedSchema;
switch(avroType) {
case RECORD:
AvroRecordSchema recordSchema = new AvroRecordSchema(codeLocation, schemaSimpleName, contextNamespace, doc, extraProps);
JsonArrayExt fieldsNode = getRequiredArray(objectNode, "fields", () -> "all avro records must have fields");
List<AvroSchemaField> fields = new ArrayList<>(fieldsNode.size());
for (int fieldNum = 0; fieldNum < fieldsNode.size(); fieldNum++) {
// !=null
JsonValueExt fieldDeclNode = (JsonValueExt) fieldsNode.get(fieldNum);
JsonValue.ValueType fieldNodeType = fieldDeclNode.getValueType();
if (fieldNodeType != JsonValue.ValueType.OBJECT) {
throw new AvroSyntaxException("field " + fieldNum + " for record " + schemaSimpleName + " at " + fieldDeclNode.getStartLocation() + " expected to be an OBJECT, not a " + JsonPUtil.describe(fieldNodeType) + " (" + fieldDeclNode + ")");
}
TextLocation fieldStartLocation = Util.convertLocation(fieldDeclNode.getStartLocation());
TextLocation fieldEndLocation = Util.convertLocation(fieldDeclNode.getEndLocation());
CodeLocation fieldCodeLocation = new CodeLocation(context.getUri(), fieldStartLocation, fieldEndLocation);
JsonObjectExt fieldDecl = (JsonObjectExt) fieldDeclNode;
Located<String> fieldName = getRequiredString(fieldDecl, "name", () -> "all record fields must have a name");
JsonValueExt fieldTypeNode = getRequiredNode(fieldDecl, "type", () -> "all record fields must have a type");
SchemaOrRef fieldSchema = parseSchemaDeclOrRef(fieldTypeNode, context, false);
JsonValueExt fieldDefaultValueNode = fieldDecl.get("default");
AvroLiteral defaultValue = null;
if (fieldDefaultValueNode != null) {
if (fieldSchema.isResolved()) {
LiteralOrIssue defaultValurOrIssue = parseLiteral(fieldDefaultValueNode, fieldSchema.getSchema(), fieldName.getValue(), context);
if (defaultValurOrIssue.getIssue() == null) {
defaultValue = defaultValurOrIssue.getLiteral();
}
// TODO - handle issues
} else {
// TODO - implement delayed default value parsing
throw new UnsupportedOperationException("delayed parsing of default value for " + fieldName.getValue() + " TBD");
}
}
LinkedHashMap<String, JsonValueExt> props = parseExtraProps(fieldDecl, CORE_FIELD_PROPERTIES);
JsonPropertiesContainer propsContainer = props.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(props);
AvroSchemaField field = new AvroSchemaField(fieldCodeLocation, fieldName.getValue(), null, fieldSchema, defaultValue, propsContainer);
fields.add(field);
}
recordSchema.setFields(fields);
namedSchema = recordSchema;
break;
case ENUM:
JsonArrayExt symbolsNode = getRequiredArray(objectNode, "symbols", () -> "all avro enums must have symbols");
List<String> symbols = new ArrayList<>(symbolsNode.size());
for (int ordinal = 0; ordinal < symbolsNode.size(); ordinal++) {
JsonValueExt symbolNode = (JsonValueExt) symbolsNode.get(ordinal);
JsonValue.ValueType symbolNodeType = symbolNode.getValueType();
if (symbolNodeType != JsonValue.ValueType.STRING) {
throw new AvroSyntaxException("symbol " + ordinal + " for enum " + schemaSimpleName + " at " + symbolNode.getStartLocation() + " expected to be a STRING, not a " + JsonPUtil.describe(symbolNodeType) + " (" + symbolNode + ")");
}
symbols.add(symbolNode.toString());
}
String defaultSymbol = null;
Located<String> defaultStr = getOptionalString(objectNode, "default");
if (defaultStr != null) {
defaultSymbol = defaultStr.getValue();
if (!symbols.contains(defaultSymbol)) {
context.addIssue(AvscIssues.badEnumDefaultValue(locationOf(context.getUri(), defaultStr), defaultSymbol, schemaSimpleName, symbols));
// TODO - support "fixing" by selecting 1st symbol as default?
defaultSymbol = null;
}
}
namedSchema = new AvroEnumSchema(codeLocation, schemaSimpleName, contextNamespace, doc, symbols, defaultSymbol, extraProps);
break;
case FIXED:
JsonValueExt sizeNode = getRequiredNode(objectNode, "size", () -> "fixed types must have a size property");
if (sizeNode.getValueType() != JsonValue.ValueType.NUMBER || !(((JsonNumberExt) sizeNode).isIntegral())) {
throw new AvroSyntaxException("size for fixed " + schemaSimpleName + " at " + sizeNode.getStartLocation() + " expected to be an INTEGER, not a " + JsonPUtil.describe(sizeNode.getValueType()) + " (" + sizeNode + ")");
}
int fixedSize = ((JsonNumberExt) sizeNode).intValue();
Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(objectNode, context, avroType, codeLocation);
if (logicalTypeResult.hasIssues()) {
context.addIssues(logicalTypeResult.getIssues());
}
namedSchema = new AvroFixedSchema(codeLocation, schemaSimpleName, contextNamespace, doc, fixedSize, logicalTypeResult.getData(), extraProps);
break;
default:
throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
}
if (namespaceChanged) {
context.popNamespace();
}
return namedSchema;
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonNumberExt in project avro-util by linkedin.
the class AvscParser method getOptionalInteger.
private Located<Integer> getOptionalInteger(JsonObjectExt node, String prop, AvscFileParseContext context) {
JsonValueExt val = node.get(prop);
if (val == null) {
return null;
}
JsonValue.ValueType valType = val.getValueType();
if (valType != JsonValue.ValueType.NUMBER) {
context.addIssue(AvscIssues.badPropertyType(prop, locationOf(context.getUri(), val), val, valType.name(), JsonValue.ValueType.NUMBER.name()));
return null;
}
JsonNumberExt numberNode = (JsonNumberExt) val;
if (!numberNode.isIntegral()) {
context.addIssue(AvscIssues.badPropertyType(prop, locationOf(context.getUri(), val), val, "floating point value", "integer"));
return null;
}
return new Located<>(numberNode.intValueExact(), Util.convertLocation(val.getStartLocation()));
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonNumberExt in project avro-util by linkedin.
the class AvscParser method parseLiteral.
private LiteralOrIssue parseLiteral(JsonValueExt literalNode, AvroSchema schema, String fieldName, AvscFileParseContext context) {
AvroType avroType = schema.type();
JsonValue.ValueType jsonType = literalNode.getValueType();
AvscIssue issue;
BigInteger bigIntegerValue;
BigDecimal bigDecimalValue;
byte[] bytes;
switch(avroType) {
case NULL:
if (jsonType != JsonValue.ValueType.NULL) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroNullLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode)));
case BOOLEAN:
if (jsonType != JsonValue.ValueType.FALSE && jsonType != JsonValue.ValueType.TRUE) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
boolean boolValue = jsonType == JsonValue.ValueType.TRUE;
return new LiteralOrIssue(new AvroBooleanLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), boolValue));
case INT:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt intNode = (JsonNumberExt) literalNode;
if (!intNode.isIntegral()) {
// TODO - be more specific about this error (int vs float)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
bigIntegerValue = intNode.bigIntegerValue();
if (bigIntegerValue.compareTo(MAX_INT) > 0 || bigIntegerValue.compareTo(MIN_INT) < 0) {
// TODO - be more specific about this error (out of signed int range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroIntegerLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.intValueExact()));
case LONG:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt longNode = (JsonNumberExt) literalNode;
if (!longNode.isIntegral()) {
// TODO - be more specific about this error (long vs float)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
bigIntegerValue = longNode.bigIntegerValue();
if (bigIntegerValue.compareTo(MAX_LONG) > 0 || bigIntegerValue.compareTo(MIN_LONG) < 0) {
// TODO - be more specific about this error (out of signed long range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroLongLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.longValueExact()));
case FLOAT:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt floatNode = (JsonNumberExt) literalNode;
bigDecimalValue = floatNode.bigDecimalValue();
if (bigDecimalValue.compareTo(MAX_FLOAT) > 0) {
// TODO - be more specific about this error (out of float range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroFloatLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.floatValue()));
case DOUBLE:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt doubleNode = (JsonNumberExt) literalNode;
bigDecimalValue = doubleNode.bigDecimalValue();
if (bigDecimalValue.compareTo(MAX_DOUBLE) > 0) {
// TODO - be more specific about this error (out of double range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroDoubleLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.doubleValue()));
case BYTES:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonStringExt bytesNode = (JsonStringExt) literalNode;
// spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
bytes = bytesNode.getString().getBytes(StandardCharsets.ISO_8859_1);
return new LiteralOrIssue(new AvroBytesLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bytes));
case FIXED:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
JsonStringExt fixedNode = (JsonStringExt) literalNode;
// spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
bytes = fixedNode.getString().getBytes(StandardCharsets.ISO_8859_1);
if (bytes.length != fixedSchema.getSize()) {
// TODO - be more specific about this error (wrong length)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroFixedLiteral(fixedSchema, locationOf(context.getUri(), literalNode), bytes));
case STRING:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonStringExt stringNode = (JsonStringExt) literalNode;
return new LiteralOrIssue(new AvroStringLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), stringNode.getString()));
case ENUM:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
JsonStringExt enumNode = (JsonStringExt) literalNode;
String enumValue = enumNode.getString();
if (!enumSchema.getSymbols().contains(enumValue)) {
// TODO - be more specific about this error (unknown symbol)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroEnumLiteral(enumSchema, locationOf(context.getUri(), literalNode), enumValue));
case ARRAY:
if (jsonType != JsonValue.ValueType.ARRAY) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroArraySchema arraySchema = (AvroArraySchema) schema;
AvroSchema valueSchema = arraySchema.getValueSchema();
JsonArrayExt arrayNode = (JsonArrayExt) literalNode;
ArrayList<AvroLiteral> values = new ArrayList<>(arrayNode.size());
for (int i = 0; i < arrayNode.size(); i++) {
JsonValueExt valueNode = (JsonValueExt) arrayNode.get(i);
LiteralOrIssue value = parseLiteral(valueNode, valueSchema, fieldName, context);
// issues parsing any member value mean a failure to parse the array as a whole
if (value.getIssue() != null) {
// TODO - be more specific about this error (unparsable array element i)
// TODO - add "causedBy" to AvscIssue and use it here
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
values.add(value.getLiteral());
}
return new LiteralOrIssue(new AvroArrayLiteral(arraySchema, locationOf(context.getUri(), literalNode), values));
default:
throw new UnsupportedOperationException("dont know how to parse a " + avroType + " at " + literalNode.getStartLocation() + " out of a " + literalNode.getValueType() + " (" + literalNode + ")");
}
}
use of com.linkedin.avroutil1.parser.jsonpext.JsonNumberExt in project avro-util by linkedin.
the class Util method convertJsonValue.
/**
* converts json values to:
* <ul>
* <li>{@link com.linkedin.avroutil1.model.JsonPropertiesContainer#NULL_VALUE} for json nulls</li>
* <li>Booleans for json boolean values</li>
* <li>BigDecimals for json numeric values</li>
* <li>java.lang.Strings for json strings</li>
* <li>
* java.util.List<Object>s for json arrays, where individual members
* are anything on this list.
* </li>
* <li>
* java.util.LinkedHashMap<String,Object>s for json objects, where
* keys are strings, values are anything on this list, and property order
* is preserved
* </li>
* </ul>
* @param jsonValue a json value
* @return value converted according to the doc
*/
public static Object convertJsonValue(JsonValueExt jsonValue) {
if (jsonValue == null) {
throw new IllegalArgumentException("jsonValue cannot be null");
}
JsonValue.ValueType jsonType = jsonValue.getValueType();
switch(jsonType) {
case NULL:
return JsonPropertiesContainer.NULL_VALUE;
case TRUE:
return Boolean.TRUE;
case FALSE:
return Boolean.FALSE;
case NUMBER:
return ((JsonNumberExt) jsonValue).bigDecimalValue();
case STRING:
return ((JsonStringExt) jsonValue).getString();
case ARRAY:
JsonArrayExt jsonArray = (JsonArrayExt) jsonValue;
List<Object> list = new ArrayList<>(jsonArray.size());
for (JsonValue val : jsonArray) {
list.add(convertJsonValue((JsonValueExt) val));
}
return list;
case OBJECT:
JsonObjectExt jsonObj = (JsonObjectExt) jsonValue;
Set<Map.Entry<String, JsonValue>> entries = jsonObj.entrySet();
LinkedHashMap<String, Object> map = new LinkedHashMap<>(entries.size());
for (Map.Entry<String, JsonValue> entry : entries) {
String key = entry.getKey();
JsonValueExt val = (JsonValueExt) entry.getValue();
map.put(key, convertJsonValue(val));
}
return map;
default:
throw new IllegalStateException("unhandled json type " + jsonType + " for " + jsonValue + " at " + jsonValue.getStartLocation());
}
}
Aggregations