use of com.linkedin.avroutil1.model.AvroFixedSchema in project avro-util by linkedin.
the class SpecificRecordClassGeneratorTest method testSimpleFixedWithHugeDoc.
@Test
public void testSimpleFixedWithHugeDoc() throws Exception {
String avsc = TestUtil.load("schemas/SimpleFixedWithHugeDoc.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroFixedSchema fixedSchema = (AvroFixedSchema) result.getTopLevelSchema();
Assert.assertNotNull(fixedSchema);
JavaFileObject javaSourceFile = generator.generateSpecificRecordClass(fixedSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaSourceFile);
}
use of com.linkedin.avroutil1.model.AvroFixedSchema in project avro-util by linkedin.
the class SpecificRecordClassGeneratorTest method testSimpleFixed.
@Test
public void testSimpleFixed() throws Exception {
String avsc = TestUtil.load("schemas/SimpleFixed.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroFixedSchema fixedSchema = (AvroFixedSchema) result.getTopLevelSchema();
Assert.assertNotNull(fixedSchema);
JavaFileObject javaSourceFile = generator.generateSpecificRecordClass(fixedSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaSourceFile);
}
use of com.linkedin.avroutil1.model.AvroFixedSchema in project avro-util by linkedin.
the class AvscParserTest method testSimpleParse.
@Test
public void testSimpleParse() throws Exception {
String avsc = TestUtil.load("schemas/TestRecord.avsc");
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroSchema schema = result.getTopLevelSchema();
Assert.assertNotNull(schema);
Assert.assertEquals(schema.type(), AvroType.RECORD);
AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
Assert.assertEquals(recordSchema.getFullName(), "com.acme.TestRecord");
List<AvroSchemaField> fields = recordSchema.getFields();
Assert.assertNotNull(fields);
Assert.assertEquals(fields.size(), 11);
Assert.assertEquals(fields.get(0).getPosition(), 0);
Assert.assertEquals(fields.get(0).getName(), "booleanField");
Assert.assertEquals(fields.get(0).getSchema().type(), AvroType.BOOLEAN);
Assert.assertEquals(fields.get(1).getPosition(), 1);
Assert.assertEquals(fields.get(1).getName(), "intField");
Assert.assertEquals(fields.get(1).getSchema().type(), AvroType.INT);
Assert.assertEquals(fields.get(2).getPosition(), 2);
Assert.assertEquals(fields.get(2).getName(), "longField");
Assert.assertEquals(fields.get(2).getSchema().type(), AvroType.LONG);
Assert.assertEquals(fields.get(3).getPosition(), 3);
Assert.assertEquals(fields.get(3).getName(), "floatField");
Assert.assertEquals(fields.get(3).getSchema().type(), AvroType.FLOAT);
Assert.assertEquals(fields.get(4).getPosition(), 4);
Assert.assertEquals(fields.get(4).getName(), "doubleField");
Assert.assertEquals(fields.get(4).getSchema().type(), AvroType.DOUBLE);
Assert.assertEquals(fields.get(5).getPosition(), 5);
Assert.assertEquals(fields.get(5).getName(), "bytesField");
Assert.assertEquals(fields.get(5).getSchema().type(), AvroType.BYTES);
Assert.assertEquals(fields.get(6).getPosition(), 6);
Assert.assertEquals(fields.get(6).getName(), "stringField");
Assert.assertEquals(fields.get(6).getSchema().type(), AvroType.STRING);
Assert.assertEquals(fields.get(7).getPosition(), 7);
Assert.assertEquals(fields.get(7).getName(), "enumField");
Assert.assertEquals(fields.get(7).getSchema().type(), AvroType.ENUM);
AvroEnumSchema simpleEnumSchema = (AvroEnumSchema) fields.get(7).getSchema();
Assert.assertEquals(simpleEnumSchema.getFullName(), "innerNamespace.SimpleEnum");
Assert.assertEquals(simpleEnumSchema.getSymbols(), Arrays.asList("A", "B", "C"));
Assert.assertEquals(fields.get(8).getPosition(), 8);
Assert.assertEquals(fields.get(8).getName(), "fixedField");
Assert.assertEquals(fields.get(8).getSchema().type(), AvroType.FIXED);
Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getFullName(), "com.acme.SimpleFixed");
Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getSize(), 7);
Assert.assertEquals(fields.get(9).getPosition(), 9);
Assert.assertEquals(fields.get(9).getName(), "strArrayField");
Assert.assertEquals(fields.get(9).getSchema().type(), AvroType.ARRAY);
Assert.assertEquals(((AvroArraySchema) fields.get(9).getSchema()).getValueSchema().type(), AvroType.NULL);
Assert.assertEquals(fields.get(10).getPosition(), 10);
Assert.assertEquals(fields.get(10).getName(), "enumMapField");
Assert.assertEquals(fields.get(10).getSchema().type(), AvroType.MAP);
AvroSchema mapValueSchema = ((AvroMapSchema) fields.get(10).getSchema()).getValueSchema();
Assert.assertSame(mapValueSchema, simpleEnumSchema);
}
use of com.linkedin.avroutil1.model.AvroFixedSchema in project avro-util by linkedin.
the class AvscParser method parseNamedSchema.
private AvroNamedSchema parseNamedSchema(JsonObjectExt objectNode, AvscFileParseContext context, AvroType avroType, CodeLocation codeLocation, JsonPropertiesContainer extraProps) {
AvroName schemaName = parseSchemaName(objectNode, context, avroType);
List<AvroName> aliases = parseAliases(objectNode, context, avroType, schemaName);
// technically the avro spec does not allow "doc" on type fixed, but screw that
Located<String> docStr = getOptionalString(objectNode, "doc");
String doc = docStr != null ? docStr.getValue() : null;
boolean namespaceChanged = false;
// check if context namespace changed
if (!context.getCurrentNamespace().equals(schemaName.getNamespace())) {
context.pushNamespace(schemaName.getNamespace());
namespaceChanged = true;
}
AvroNamedSchema namedSchema;
switch(avroType) {
case RECORD:
AvroRecordSchema recordSchema = new AvroRecordSchema(codeLocation, schemaName, aliases, doc, extraProps);
JsonArrayExt fieldsNode = getRequiredArray(objectNode, "fields", () -> "all avro records must have fields");
List<AvroSchemaField> fields = new ArrayList<>(fieldsNode.size());
for (int fieldNum = 0; fieldNum < fieldsNode.size(); fieldNum++) {
// !=null
JsonValueExt fieldDeclNode = (JsonValueExt) fieldsNode.get(fieldNum);
JsonValue.ValueType fieldNodeType = fieldDeclNode.getValueType();
if (fieldNodeType != JsonValue.ValueType.OBJECT) {
throw new AvroSyntaxException("field " + fieldNum + " for record " + schemaName.getSimpleName() + " at " + fieldDeclNode.getStartLocation() + " expected to be an OBJECT, not a " + JsonPUtil.describe(fieldNodeType) + " (" + fieldDeclNode + ")");
}
TextLocation fieldStartLocation = Util.convertLocation(fieldDeclNode.getStartLocation());
TextLocation fieldEndLocation = Util.convertLocation(fieldDeclNode.getEndLocation());
CodeLocation fieldCodeLocation = new CodeLocation(context.getUri(), fieldStartLocation, fieldEndLocation);
JsonObjectExt fieldDecl = (JsonObjectExt) fieldDeclNode;
Located<String> fieldName = getRequiredString(fieldDecl, "name", () -> "all record fields must have a name");
JsonValueExt fieldTypeNode = getRequiredNode(fieldDecl, "type", () -> "all record fields must have a type");
Located<String> locatedDocField = getOptionalString(fieldDecl, "doc");
String docField = locatedDocField == null ? null : locatedDocField.getValue();
SchemaOrRef fieldSchema = parseSchemaDeclOrRef(fieldTypeNode, context, false);
JsonValueExt fieldDefaultValueNode = fieldDecl.get("default");
AvroLiteral defaultValue = null;
if (fieldDefaultValueNode != null) {
if (fieldSchema.isResolved()) {
LiteralOrIssue defaultValurOrIssue = parseLiteral(fieldDefaultValueNode, fieldSchema.getSchema(), fieldName.getValue(), context);
if (defaultValurOrIssue.getIssue() == null) {
defaultValue = defaultValurOrIssue.getLiteral();
}
// TODO - handle issues
} else {
// TODO - implement delayed default value parsing
throw new UnsupportedOperationException("delayed parsing of default value for " + fieldName.getValue() + " TBD");
}
}
LinkedHashMap<String, JsonValueExt> props = parseExtraProps(fieldDecl, CORE_FIELD_PROPERTIES);
JsonPropertiesContainer propsContainer = props.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(props);
AvroSchemaField field = new AvroSchemaField(fieldCodeLocation, fieldName.getValue(), docField, fieldSchema, defaultValue, propsContainer);
fields.add(field);
}
recordSchema.setFields(fields);
namedSchema = recordSchema;
break;
case ENUM:
JsonArrayExt symbolsNode = getRequiredArray(objectNode, "symbols", () -> "all avro enums must have symbols");
List<String> symbols = new ArrayList<>(symbolsNode.size());
for (int ordinal = 0; ordinal < symbolsNode.size(); ordinal++) {
JsonValueExt symbolNode = (JsonValueExt) symbolsNode.get(ordinal);
JsonValue.ValueType symbolNodeType = symbolNode.getValueType();
if (symbolNodeType != JsonValue.ValueType.STRING) {
throw new AvroSyntaxException("symbol " + ordinal + " for enum " + schemaName.getSimpleName() + " at " + symbolNode.getStartLocation() + " expected to be a STRING, not a " + JsonPUtil.describe(symbolNodeType) + " (" + symbolNode + ")");
}
symbols.add(symbolNode.toString());
}
String defaultSymbol = null;
Located<String> defaultStr = getOptionalString(objectNode, "default");
if (defaultStr != null) {
defaultSymbol = defaultStr.getValue();
if (!symbols.contains(defaultSymbol)) {
context.addIssue(AvscIssues.badEnumDefaultValue(locationOf(context.getUri(), defaultStr), defaultSymbol, schemaName.getSimpleName(), symbols));
// TODO - support "fixing" by selecting 1st symbol as default?
defaultSymbol = null;
}
}
namedSchema = new AvroEnumSchema(codeLocation, schemaName, aliases, doc, symbols, defaultSymbol, extraProps);
break;
case FIXED:
JsonValueExt sizeNode = getRequiredNode(objectNode, "size", () -> "fixed types must have a size property");
if (sizeNode.getValueType() != JsonValue.ValueType.NUMBER || !(((JsonNumberExt) sizeNode).isIntegral())) {
throw new AvroSyntaxException("size for fixed " + schemaName.getSimpleName() + " at " + sizeNode.getStartLocation() + " expected to be an INTEGER, not a " + JsonPUtil.describe(sizeNode.getValueType()) + " (" + sizeNode + ")");
}
int fixedSize = ((JsonNumberExt) sizeNode).intValue();
Parsed<AvroLogicalType> logicalTypeResult = parseLogicalType(objectNode, context, avroType, codeLocation);
if (logicalTypeResult.hasIssues()) {
context.addIssues(logicalTypeResult.getIssues());
}
namedSchema = new AvroFixedSchema(codeLocation, schemaName, aliases, doc, fixedSize, logicalTypeResult.getData(), extraProps);
break;
default:
throw new IllegalStateException("unhandled: " + avroType + " for object at " + codeLocation.getStart());
}
if (namespaceChanged) {
context.popNamespace();
}
return namedSchema;
}
use of com.linkedin.avroutil1.model.AvroFixedSchema in project avro-util by linkedin.
the class AvscSchemaWriter method writeNamedSchema.
protected JsonValue writeNamedSchema(AvroNamedSchema schema, AvscWriterContext context, AvscWriterConfig config) {
boolean seenBefore = context.schemaEncountered(schema);
if (seenBefore) {
return writeSchemaRef(schema, context, config);
}
// common parts to all named schemas
JsonObjectBuilder definitionBuilder = Json.createObjectBuilder();
AvroName extraAlias = emitSchemaName(schema, context, config, definitionBuilder);
emitSchemaAliases(schema, context, config, extraAlias, definitionBuilder);
if (schema.getDoc() != null) {
definitionBuilder.add("doc", Json.createValue(schema.getDoc()));
}
AvroType type = schema.type();
switch(type) {
case ENUM:
AvroEnumSchema enumSchema = (AvroEnumSchema) schema;
definitionBuilder.add("type", "enum");
List<String> symbols = enumSchema.getSymbols();
JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
for (String symbol : symbols) {
arrayBuilder.add(symbol);
}
definitionBuilder.add("symbols", arrayBuilder);
String defaultSymbol = enumSchema.getDefaultSymbol();
if (defaultSymbol != null) {
definitionBuilder.add("default", Json.createValue(defaultSymbol));
}
break;
case FIXED:
AvroFixedSchema fixedSchema = (AvroFixedSchema) schema;
definitionBuilder.add("type", "fixed");
definitionBuilder.add("size", Json.createValue(fixedSchema.getSize()));
break;
case RECORD:
AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
// TODO - support error types?
definitionBuilder.add("type", "record");
emitRecordFields(recordSchema, context, config, definitionBuilder);
break;
default:
throw new IllegalStateException("not expecting " + type);
}
emitJsonProperties(schema, context, config, definitionBuilder);
context.popNamingContext();
return definitionBuilder.build();
}
Aggregations