use of com.linkedin.avroutil1.model.AvroArraySchema in project avro-util by linkedin.
the class AvscSchemaWriter method writeDefaultValue.
protected JsonValue writeDefaultValue(AvroSchema fieldSchema, AvroLiteral literal) {
AvroType type = fieldSchema.type();
String temp;
switch(type) {
case NULL:
// noinspection unused (kept as a sanity check)
AvroNullLiteral nullLiteral = (AvroNullLiteral) literal;
return JsonValue.NULL;
case BOOLEAN:
AvroBooleanLiteral boolLiteral = (AvroBooleanLiteral) literal;
return boolLiteral.getValue() ? JsonValue.TRUE : JsonValue.FALSE;
case INT:
AvroIntegerLiteral intLiteral = (AvroIntegerLiteral) literal;
return Json.createValue(intLiteral.getValue());
case LONG:
AvroLongLiteral longLiteral = (AvroLongLiteral) literal;
return Json.createValue(longLiteral.getValue());
case FLOAT:
AvroFloatLiteral floatLiteral = (AvroFloatLiteral) literal;
return Json.createValue(floatLiteral.getValue());
case DOUBLE:
AvroDoubleLiteral doubleLiteral = (AvroDoubleLiteral) literal;
return Json.createValue(doubleLiteral.getValue());
case STRING:
AvroStringLiteral stringLiteral = (AvroStringLiteral) literal;
return Json.createValue(stringLiteral.getValue());
case BYTES:
AvroBytesLiteral bytesLiteral = (AvroBytesLiteral) literal;
// spec says "values for bytes and fixed fields are JSON strings, where Unicode code points
// 0-255 are mapped to unsigned 8-bit byte values 0-255", and this is how its done
temp = new String(bytesLiteral.getValue(), StandardCharsets.ISO_8859_1);
return Json.createValue(temp);
case ENUM:
AvroEnumLiteral enumLiteral = (AvroEnumLiteral) literal;
return Json.createValue(enumLiteral.getValue());
case FIXED:
AvroFixedLiteral fixedLiteral = (AvroFixedLiteral) literal;
// spec says "values for bytes and fixed fields are JSON strings, where Unicode code points
// 0-255 are mapped to unsigned 8-bit byte values 0-255", and this is how its done
temp = new String(fixedLiteral.getValue(), StandardCharsets.ISO_8859_1);
return Json.createValue(temp);
case ARRAY:
AvroArrayLiteral arrayLiteral = (AvroArrayLiteral) literal;
List<AvroLiteral> array = arrayLiteral.getValue();
AvroArraySchema arraySchema = (AvroArraySchema) arrayLiteral.getSchema();
JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
for (AvroLiteral element : array) {
arrayBuilder.add(writeDefaultValue(arraySchema.getValueSchema(), element));
}
return arrayBuilder.build();
default:
throw new UnsupportedOperationException("writing default values for " + type + " not implemented yet");
}
}
use of com.linkedin.avroutil1.model.AvroArraySchema in project avro-util by linkedin.
the class AvscParserTest method testSimpleParse.
@Test
public void testSimpleParse() throws Exception {
String avsc = TestUtil.load("schemas/TestRecord.avsc");
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroSchema schema = result.getTopLevelSchema();
Assert.assertNotNull(schema);
Assert.assertEquals(schema.type(), AvroType.RECORD);
AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
Assert.assertEquals(recordSchema.getFullName(), "com.acme.TestRecord");
List<AvroSchemaField> fields = recordSchema.getFields();
Assert.assertNotNull(fields);
Assert.assertEquals(fields.size(), 11);
Assert.assertEquals(fields.get(0).getPosition(), 0);
Assert.assertEquals(fields.get(0).getName(), "booleanField");
Assert.assertEquals(fields.get(0).getSchema().type(), AvroType.BOOLEAN);
Assert.assertEquals(fields.get(1).getPosition(), 1);
Assert.assertEquals(fields.get(1).getName(), "intField");
Assert.assertEquals(fields.get(1).getSchema().type(), AvroType.INT);
Assert.assertEquals(fields.get(2).getPosition(), 2);
Assert.assertEquals(fields.get(2).getName(), "longField");
Assert.assertEquals(fields.get(2).getSchema().type(), AvroType.LONG);
Assert.assertEquals(fields.get(3).getPosition(), 3);
Assert.assertEquals(fields.get(3).getName(), "floatField");
Assert.assertEquals(fields.get(3).getSchema().type(), AvroType.FLOAT);
Assert.assertEquals(fields.get(4).getPosition(), 4);
Assert.assertEquals(fields.get(4).getName(), "doubleField");
Assert.assertEquals(fields.get(4).getSchema().type(), AvroType.DOUBLE);
Assert.assertEquals(fields.get(5).getPosition(), 5);
Assert.assertEquals(fields.get(5).getName(), "bytesField");
Assert.assertEquals(fields.get(5).getSchema().type(), AvroType.BYTES);
Assert.assertEquals(fields.get(6).getPosition(), 6);
Assert.assertEquals(fields.get(6).getName(), "stringField");
Assert.assertEquals(fields.get(6).getSchema().type(), AvroType.STRING);
Assert.assertEquals(fields.get(7).getPosition(), 7);
Assert.assertEquals(fields.get(7).getName(), "enumField");
Assert.assertEquals(fields.get(7).getSchema().type(), AvroType.ENUM);
AvroEnumSchema simpleEnumSchema = (AvroEnumSchema) fields.get(7).getSchema();
Assert.assertEquals(simpleEnumSchema.getFullName(), "innerNamespace.SimpleEnum");
Assert.assertEquals(simpleEnumSchema.getSymbols(), Arrays.asList("A", "B", "C"));
Assert.assertEquals(fields.get(8).getPosition(), 8);
Assert.assertEquals(fields.get(8).getName(), "fixedField");
Assert.assertEquals(fields.get(8).getSchema().type(), AvroType.FIXED);
Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getFullName(), "com.acme.SimpleFixed");
Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getSize(), 7);
Assert.assertEquals(fields.get(9).getPosition(), 9);
Assert.assertEquals(fields.get(9).getName(), "strArrayField");
Assert.assertEquals(fields.get(9).getSchema().type(), AvroType.ARRAY);
Assert.assertEquals(((AvroArraySchema) fields.get(9).getSchema()).getValueSchema().type(), AvroType.NULL);
Assert.assertEquals(fields.get(10).getPosition(), 10);
Assert.assertEquals(fields.get(10).getName(), "enumMapField");
Assert.assertEquals(fields.get(10).getSchema().type(), AvroType.MAP);
AvroSchema mapValueSchema = ((AvroMapSchema) fields.get(10).getSchema()).getValueSchema();
Assert.assertSame(mapValueSchema, simpleEnumSchema);
}
use of com.linkedin.avroutil1.model.AvroArraySchema in project avro-util by linkedin.
the class AvscSchemaWriter method writeSchema.
protected JsonValue writeSchema(AvroSchema schema, AvscWriterContext context, AvscWriterConfig config) {
AvroType type = schema.type();
JsonObjectBuilder definitionBuilder;
switch(type) {
case ENUM:
case FIXED:
case RECORD:
return writeNamedSchema((AvroNamedSchema) schema, context, config);
case ARRAY:
AvroArraySchema arraySchema = (AvroArraySchema) schema;
definitionBuilder = Json.createObjectBuilder();
definitionBuilder.add("type", "array");
definitionBuilder.add("items", writeSchema(arraySchema.getValueSchema(), context, config));
emitJsonProperties(schema, context, config, definitionBuilder);
return definitionBuilder.build();
case MAP:
AvroMapSchema mapSchema = (AvroMapSchema) schema;
definitionBuilder = Json.createObjectBuilder();
definitionBuilder.add("type", "map");
definitionBuilder.add("values", writeSchema(mapSchema.getValueSchema(), context, config));
emitJsonProperties(schema, context, config, definitionBuilder);
return definitionBuilder.build();
case UNION:
AvroUnionSchema unionSchema = (AvroUnionSchema) schema;
JsonArrayBuilder unionBuilder = Json.createArrayBuilder();
for (SchemaOrRef unionBranch : unionSchema.getTypes()) {
// will throw if unresolved ref
AvroSchema branchSchema = unionBranch.getSchema();
unionBuilder.add(writeSchema(branchSchema, context, config));
}
return unionBuilder.build();
default:
AvroPrimitiveSchema primitiveSchema = (AvroPrimitiveSchema) schema;
if (!primitiveSchema.hasProperties()) {
return Json.createValue(primitiveSchema.type().name().toLowerCase(Locale.ROOT));
}
definitionBuilder = Json.createObjectBuilder();
definitionBuilder.add("type", primitiveSchema.type().toTypeName());
emitJsonProperties(primitiveSchema, context, config, definitionBuilder);
return definitionBuilder.build();
}
}
use of com.linkedin.avroutil1.model.AvroArraySchema in project avro-util by linkedin.
the class AvscFileParseContext method resolveReferences.
private void resolveReferences(AvroSchema schema) {
AvroType type = schema.type();
switch(type) {
case RECORD:
AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
List<AvroSchemaField> fields = recordSchema.getFields();
for (AvroSchemaField field : fields) {
SchemaOrRef fieldSchema = field.getSchemaOrRef();
resolveReferences(fieldSchema);
}
break;
case UNION:
AvroUnionSchema unionSchema = (AvroUnionSchema) schema;
List<SchemaOrRef> types = unionSchema.getTypes();
for (SchemaOrRef unionType : types) {
resolveReferences(unionType);
}
break;
case ARRAY:
AvroArraySchema arraySchema = (AvroArraySchema) schema;
SchemaOrRef arrayValuesType = arraySchema.getValueSchemaOrRef();
resolveReferences(arrayValuesType);
break;
case MAP:
AvroMapSchema mapSchema = (AvroMapSchema) schema;
SchemaOrRef mapValuesType = mapSchema.getValueSchemaOrRef();
resolveReferences(mapValuesType);
break;
default:
break;
}
}
use of com.linkedin.avroutil1.model.AvroArraySchema in project avro-util by linkedin.
the class AvscParser method parseLiteral.
private LiteralOrIssue parseLiteral(JsonValueExt literalNode, AvroSchema schema, String fieldName, AvscFileParseContext context) {
AvroType avroType = schema.type();
JsonValue.ValueType jsonType = literalNode.getValueType();
AvscIssue issue;
BigInteger bigIntegerValue;
BigDecimal bigDecimalValue;
byte[] bytes;
switch(avroType) {
case NULL:
if (jsonType != JsonValue.ValueType.NULL) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroNullLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode)));
case BOOLEAN:
if (jsonType != JsonValue.ValueType.FALSE && jsonType != JsonValue.ValueType.TRUE) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
boolean boolValue = jsonType == JsonValue.ValueType.TRUE;
return new LiteralOrIssue(new AvroBooleanLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), boolValue));
case INT:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt intNode = (JsonNumberExt) literalNode;
if (!intNode.isIntegral()) {
// TODO - be more specific about this error (int vs float)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
bigIntegerValue = intNode.bigIntegerValue();
if (bigIntegerValue.compareTo(MAX_INT) > 0 || bigIntegerValue.compareTo(MIN_INT) < 0) {
// TODO - be more specific about this error (out of signed int range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroIntegerLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.intValueExact()));
case LONG:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt longNode = (JsonNumberExt) literalNode;
if (!longNode.isIntegral()) {
// TODO - be more specific about this error (long vs float)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
bigIntegerValue = longNode.bigIntegerValue();
if (bigIntegerValue.compareTo(MAX_LONG) > 0 || bigIntegerValue.compareTo(MIN_LONG) < 0) {
// TODO - be more specific about this error (out of signed long range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroLongLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigIntegerValue.longValueExact()));
case FLOAT:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt floatNode = (JsonNumberExt) literalNode;
bigDecimalValue = floatNode.bigDecimalValue();
if (bigDecimalValue.compareTo(MAX_FLOAT) > 0) {
// TODO - be more specific about this error (out of float range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroFloatLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.floatValue()));
case DOUBLE:
if (jsonType != JsonValue.ValueType.NUMBER) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonNumberExt doubleNode = (JsonNumberExt) literalNode;
bigDecimalValue = doubleNode.bigDecimalValue();
if (bigDecimalValue.compareTo(MAX_DOUBLE) > 0) {
// TODO - be more specific about this error (out of double range)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroDoubleLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bigDecimalValue.doubleValue()));
case BYTES:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonStringExt bytesNode = (JsonStringExt) literalNode;
// spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
bytes = bytesNode.getString().getBytes(StandardCharsets.ISO_8859_1);
return new LiteralOrIssue(new AvroBytesLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), bytes));
case FIXED:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
JsonStringExt fixedNode = (JsonStringExt) literalNode;
// spec says "strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255"
bytes = fixedNode.getString().getBytes(StandardCharsets.ISO_8859_1);
if (bytes.length != fixedSchema.getSize()) {
// TODO - be more specific about this error (wrong length)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroFixedLiteral(fixedSchema, locationOf(context.getUri(), literalNode), bytes));
case STRING:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
JsonStringExt stringNode = (JsonStringExt) literalNode;
return new LiteralOrIssue(new AvroStringLiteral((AvroPrimitiveSchema) schema, locationOf(context.getUri(), literalNode), stringNode.getString()));
case ENUM:
if (jsonType != JsonValue.ValueType.STRING) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
JsonStringExt enumNode = (JsonStringExt) literalNode;
String enumValue = enumNode.getString();
if (!enumSchema.getSymbols().contains(enumValue)) {
// TODO - be more specific about this error (unknown symbol)
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
return new LiteralOrIssue(new AvroEnumLiteral(enumSchema, locationOf(context.getUri(), literalNode), enumValue));
case ARRAY:
if (jsonType != JsonValue.ValueType.ARRAY) {
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
AvroArraySchema arraySchema = (AvroArraySchema) schema;
AvroSchema valueSchema = arraySchema.getValueSchema();
JsonArrayExt arrayNode = (JsonArrayExt) literalNode;
ArrayList<AvroLiteral> values = new ArrayList<>(arrayNode.size());
for (int i = 0; i < arrayNode.size(); i++) {
JsonValueExt valueNode = (JsonValueExt) arrayNode.get(i);
LiteralOrIssue value = parseLiteral(valueNode, valueSchema, fieldName, context);
// issues parsing any member value mean a failure to parse the array as a whole
if (value.getIssue() != null) {
// TODO - be more specific about this error (unparsable array element i)
// TODO - add "causedBy" to AvscIssue and use it here
issue = AvscIssues.badFieldDefaultValue(locationOf(context.getUri(), literalNode), literalNode.toString(), avroType, fieldName);
context.addIssue(issue);
return new LiteralOrIssue(issue);
}
values.add(value.getLiteral());
}
return new LiteralOrIssue(new AvroArrayLiteral(arraySchema, locationOf(context.getUri(), literalNode), values));
default:
throw new UnsupportedOperationException("dont know how to parse a " + avroType + " at " + literalNode.getStartLocation() + " out of a " + literalNode.getValueType() + " (" + literalNode + ")");
}
}
Aggregations