Examples with AvroSchema com.linkedin.avroutil1.model.AvroSchema used on opensource projects

Example 6 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscParserTest method testSimpleParse.

@Test
public void testSimpleParse() throws Exception {
    String avsc = TestUtil.load("schemas/TestRecord.avsc");
    AvscParser parser = new AvscParser();
    AvscParseResult result = parser.parse(avsc);
    Assert.assertNull(result.getParseError());
    AvroSchema schema = result.getTopLevelSchema();
    Assert.assertNotNull(schema);
    Assert.assertEquals(schema.type(), AvroType.RECORD);
    AvroRecordSchema recordSchema = (AvroRecordSchema) schema;
    Assert.assertEquals(recordSchema.getFullName(), "com.acme.TestRecord");
    List<AvroSchemaField> fields = recordSchema.getFields();
    Assert.assertNotNull(fields);
    Assert.assertEquals(fields.size(), 11);
    Assert.assertEquals(fields.get(0).getPosition(), 0);
    Assert.assertEquals(fields.get(0).getName(), "booleanField");
    Assert.assertEquals(fields.get(0).getSchema().type(), AvroType.BOOLEAN);
    Assert.assertEquals(fields.get(1).getPosition(), 1);
    Assert.assertEquals(fields.get(1).getName(), "intField");
    Assert.assertEquals(fields.get(1).getSchema().type(), AvroType.INT);
    Assert.assertEquals(fields.get(2).getPosition(), 2);
    Assert.assertEquals(fields.get(2).getName(), "longField");
    Assert.assertEquals(fields.get(2).getSchema().type(), AvroType.LONG);
    Assert.assertEquals(fields.get(3).getPosition(), 3);
    Assert.assertEquals(fields.get(3).getName(), "floatField");
    Assert.assertEquals(fields.get(3).getSchema().type(), AvroType.FLOAT);
    Assert.assertEquals(fields.get(4).getPosition(), 4);
    Assert.assertEquals(fields.get(4).getName(), "doubleField");
    Assert.assertEquals(fields.get(4).getSchema().type(), AvroType.DOUBLE);
    Assert.assertEquals(fields.get(5).getPosition(), 5);
    Assert.assertEquals(fields.get(5).getName(), "bytesField");
    Assert.assertEquals(fields.get(5).getSchema().type(), AvroType.BYTES);
    Assert.assertEquals(fields.get(6).getPosition(), 6);
    Assert.assertEquals(fields.get(6).getName(), "stringField");
    Assert.assertEquals(fields.get(6).getSchema().type(), AvroType.STRING);
    Assert.assertEquals(fields.get(7).getPosition(), 7);
    Assert.assertEquals(fields.get(7).getName(), "enumField");
    Assert.assertEquals(fields.get(7).getSchema().type(), AvroType.ENUM);
    AvroEnumSchema simpleEnumSchema = (AvroEnumSchema) fields.get(7).getSchema();
    Assert.assertEquals(simpleEnumSchema.getFullName(), "innerNamespace.SimpleEnum");
    Assert.assertEquals(simpleEnumSchema.getSymbols(), Arrays.asList("A", "B", "C"));
    Assert.assertEquals(fields.get(8).getPosition(), 8);
    Assert.assertEquals(fields.get(8).getName(), "fixedField");
    Assert.assertEquals(fields.get(8).getSchema().type(), AvroType.FIXED);
    Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getFullName(), "com.acme.SimpleFixed");
    Assert.assertEquals(((AvroFixedSchema) fields.get(8).getSchema()).getSize(), 7);
    Assert.assertEquals(fields.get(9).getPosition(), 9);
    Assert.assertEquals(fields.get(9).getName(), "strArrayField");
    Assert.assertEquals(fields.get(9).getSchema().type(), AvroType.ARRAY);
    Assert.assertEquals(((AvroArraySchema) fields.get(9).getSchema()).getValueSchema().type(), AvroType.NULL);
    Assert.assertEquals(fields.get(10).getPosition(), 10);
    Assert.assertEquals(fields.get(10).getName(), "enumMapField");
    Assert.assertEquals(fields.get(10).getSchema().type(), AvroType.MAP);
    AvroSchema mapValueSchema = ((AvroMapSchema) fields.get(10).getSchema()).getValueSchema();
    Assert.assertSame(mapValueSchema, simpleEnumSchema);
}

Example 7 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscParser method parseComplexSchema.

private SchemaOrRef parseComplexSchema(JsonObjectExt objectNode, AvscFileParseContext context, boolean topLevel) {
    CodeLocation codeLocation = locationOf(context.getUri(), objectNode);
    Located<String> typeStr = getRequiredString(objectNode, "type", () -> "it is a schema declaration");
    AvroType avroType = AvroType.fromTypeName(typeStr.getValue());
    if (avroType == null) {
        throw new AvroSyntaxException("unknown avro type \"" + typeStr.getValue() + "\" at " + typeStr.getLocation() + ". expecting \"record\", \"enum\" or \"fixed\"");
    }
    LinkedHashMap<String, JsonValueExt> propsMap = parseExtraProps(objectNode, CORE_SCHEMA_PROPERTIES);
    JsonPropertiesContainer props = propsMap.isEmpty() ? JsonPropertiesContainer.EMPTY : new JsonPropertiesContainerImpl(propsMap);
    AvroSchema definedSchema;
    if (avroType.isNamed()) {
        definedSchema = parseNamedSchema(objectNode, context, avroType, codeLocation, props);
    } else if (avroType.isCollection()) {
        definedSchema = parseCollectionSchema(objectNode, context, avroType, codeLocation, props);
    } else if (avroType.isPrimitive()) {
        definedSchema = parseDecoratedPrimitiveSchema(objectNode, context, avroType, codeLocation, props);
    } else {
        throw new IllegalStateException("unhandled avro type " + avroType + " at " + typeStr.getLocation());
    }
    context.defineSchema(definedSchema, topLevel);
    return new SchemaOrRef(codeLocation, definedSchema);
}

Example 8 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscParserTest method testParsingProperties.

@Test
public void testParsingProperties() throws Exception {
    String avsc = TestUtil.load("schemas/TestRecordWithProperties.avsc");
    AvscParser parser = new AvscParser();
    AvscParseResult result = parser.parse(avsc);
    Assert.assertNull(result.getParseError());
    AvroRecordSchema schema = (AvroRecordSchema) result.getTopLevelSchema();
    Assert.assertNotNull(schema);
    Assert.assertEquals(schema.propertyNames(), // order is important
    Arrays.asList("extraNullProp", "extraBooleanProp", "extraIntProp", "extraFloatProp", "extraStringProp", "extraArrayProp", "extraObjectProp"));
    Assert.assertNull(schema.getPropertyAsJsonLiteral("noSuchProp"));
    Assert.assertNull(schema.getPropertyAsObject("noSuchProp"));
    Assert.assertEquals(schema.getPropertyAsJsonLiteral("extraNullProp"), "null");
    Assert.assertEquals(schema.getPropertyAsObject("extraNullProp"), JsonPropertiesContainer.NULL_VALUE);
    Assert.assertEquals(schema.getPropertyAsJsonLiteral("extraBooleanProp"), "true");
    Assert.assertEquals(schema.getPropertyAsObject("extraBooleanProp"), Boolean.TRUE);
    Assert.assertEquals(schema.getPropertyAsJsonLiteral("extraIntProp"), "42");
    Assert.assertEquals(schema.getPropertyAsObject("extraIntProp"), new BigDecimal(42));
    Assert.assertEquals(schema.getPropertyAsJsonLiteral("extraFloatProp"), "4.2");
    // new BigDecimal(4.2d) is actually 4.20000000000000017763568394002504646778106689453125
    // we love floating point precision
    Assert.assertEquals(schema.getPropertyAsObject("extraFloatProp"), new BigDecimal("4.2"));
    Assert.assertEquals(schema.getPropertyAsJsonLiteral("extraStringProp"), "\"a string\"");
    Assert.assertEquals(schema.getPropertyAsObject("extraStringProp"), "a string");
    JSONAssert.assertEquals("[null, 0, false, \"wow\", {\"this\" : \"makes\", \"little\" : \"sense\"}]", schema.getPropertyAsJsonLiteral("extraArrayProp"), JSONCompareMode.STRICT);
    Assert.assertEquals(schema.getPropertyAsObject("extraArrayProp"), Arrays.asList(JsonPropertiesContainer.NULL_VALUE, new BigDecimal(0), Boolean.FALSE, "wow", new LinkedHashMap<String, Object>() {

        {
            put("this", "makes");
            put("little", "sense");
        }
    }));
    JSONAssert.assertEquals("{\"thats\": [\"all\", \"folks\"]}", schema.getPropertyAsJsonLiteral("extraObjectProp"), JSONCompareMode.STRICT);
    Assert.assertEquals(schema.getPropertyAsObject("extraObjectProp"), new LinkedHashMap<String, Object>() {

        {
            put("thats", Arrays.asList("all", "folks"));
        }
    });
    AvroSchemaField stringField = schema.getField("stringField");
    Assert.assertEquals(stringField.propertyNames(), Collections.singletonList("fieldStringProp"));
    Assert.assertEquals(stringField.getPropertyAsJsonLiteral("fieldStringProp"), "\"fieldStringValue\"");
    Assert.assertEquals(stringField.getPropertyAsObject("fieldStringProp"), "fieldStringValue");
    AvroSchema stringSchema = stringField.getSchema();
    Assert.assertEquals(stringSchema.propertyNames(), Arrays.asList("avro.java.string", "typeStringProp"));
    AvroSchema uuidSchema = schema.getField("uuidField").getSchema();
    Assert.assertEquals(uuidSchema.propertyNames(), Collections.singletonList("logicalType"));
}

Example 9 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvscSchemaWriterTest method testParsingCycle.

/**
 * given an avsc, parses and re-prints it using our code
 * and compares the result to vanilla avro.
 * @param avsc
 */
private void testParsingCycle(String avsc) {
    Schema reference = Schema.parse(avsc);
    AvscParser parser = new AvscParser();
    AvscParseResult parseResults = parser.parse(avsc);
    List<AvscIssue> parseIssues = parseResults.getIssues();
    Assert.assertTrue(parseIssues == null || parseIssues.isEmpty(), "parse issues: " + parseIssues);
    AvroSchema parsed = parseResults.getTopLevelSchema();
    Assert.assertNotNull(parsed);
    AvscSchemaWriter writer = new AvscSchemaWriter();
    AvscFile file = writer.writeSingle(parsed);
    Assert.assertNotNull(file);
    if (HelperConsts.NAMED_TYPES.contains(reference.getType())) {
        // for named schemas the file path is determined by schema name
        Assert.assertNotNull(file.getPathFromRoot());
        String expectedFileName = reference.getFullName().replaceAll("\\.", Matcher.quoteReplacement(File.separator)) + ".avsc";
        Assert.assertEquals(file.getPathFromRoot().toString(), expectedFileName);
    } else {
        // cant auto-name files containing other schema types
        Assert.assertNull(file.getPathFromRoot());
    }
    String avsc2 = file.getContents();
    Schema afterCycle = Schema.parse(avsc2);
    Assert.assertEquals(reference, afterCycle);
}

Example 10 with AvroSchema

use of com.linkedin.avroutil1.model.AvroSchema in project avro-util by linkedin.

the class AvroParseContext method resolveReferences.

public void resolveReferences() {
    sealed = true;
    // build up an index of FQCNs (also find dups)
    knownNamedSchemas = new HashMap<>(individualResults.size());
    duplicates = new HashMap<>(1);
    for (AvscParseResult singleFile : individualResults) {
        Throwable error = singleFile.getParseError();
        if (error != null) {
            // dont touch files with outright failures
            continue;
        }
        Map<String, AvroSchema> namedInFile = singleFile.getDefinedNamedSchemas();
        namedInFile.forEach((fqcn, schema) -> {
            AvscParseResult firstDefinition = knownNamedSchemas.putIfAbsent(fqcn, singleFile);
            if (firstDefinition != null) {
                // TODO - find dups in aliases as well ?
                // this is a dup
                duplicates.compute(fqcn, (k, dups) -> {
                    if (dups == null) {
                        dups = new ArrayList<>(2);
                        dups.add(firstDefinition);
                    }
                    dups.add(singleFile);
                    return dups;
                });
            }
        });
    }
    // TODO - add context-level issues for dups
    // resolve any unresolved references in individual file results from other files
    externalReferences = new ArrayList<>();
    for (AvscParseResult singleFile : individualResults) {
        List<SchemaOrRef> externalRefs = singleFile.getExternalReferences();
        for (SchemaOrRef ref : externalRefs) {
            String simpleName = ref.getRef();
            AvscParseResult simpleNameResolution = knownNamedSchemas.get(simpleName);
            AvscParseResult inheritedNameResolution = null;
            String inheritedName = ref.getInheritedName();
            if (inheritedName != null) {
                inheritedNameResolution = knownNamedSchemas.get(inheritedName);
            }
            // the inherited namespace).
            if (inheritedNameResolution != null) {
                ref.setResolvedTo(inheritedNameResolution.getDefinedNamedSchemas().get(inheritedName));
                if (simpleNameResolution != null) {
                    String msg = "ERROR: Two different schemas found for reference " + simpleName + " with inherited name " + inheritedName + ". Only one should exist.";
                    singleFile.addIssue(new AvscIssue(ref.getCodeLocation(), IssueSeverity.WARNING, msg, new IllegalStateException(msg)));
                }
            } else if (simpleNameResolution != null) {
                ref.setResolvedTo(simpleNameResolution.getDefinedNamedSchemas().get(simpleName));
            } else {
                // fqcn is unresolved in this context
                externalReferences.add(ref);
            }
        }
    }
}