Examples with SchemaParseResult com.linkedin.avroutil1.compatibility.SchemaParseResult used on opensource projects

Example 1 with SchemaParseResult

use of com.linkedin.avroutil1.compatibility.SchemaParseResult in project avro-util by linkedin.

the class CodeGenerator method parseSchemas.

private Map<String, SchemaDetails> parseSchemas(Set<File> toParse, Map<String, SchemaDetails> importableSchemas, Map<String, SchemaDetails> externalSchemas) {
    Collection<File> toTry = toParse;
    Map<String, SchemaDetails> successfullyParsed = new HashMap<>();
    int passNumber = 0;
    while (!toTry.isEmpty()) {
        boolean madeProgress = false;
        passNumber++;
        Map<File, FileParseIssue> failedFiles = new HashMap<>();
        for (File schemaFile : toTry) {
            String fileContents = readFile(schemaFile);
            SchemaParseResult result;
            try {
                // build up a collection of all known schemas to hand to avro
                List<Schema> allKnownSchemas = buildUpKnownSchemaLists(successfullyParsed, importableSchemas, externalSchemas);
                // call avro to parse our file given all the known schemas we've built up above
                result = AvroCompatibilityHelper.parse(fileContents, SchemaParseConfiguration.STRICT, allKnownSchemas);
                madeProgress = true;
            } catch (SchemaParseException parseException) {
                failedFiles.put(schemaFile, new FileParseIssue(schemaFile, parseException));
                // to next file
                continue;
            } catch (Exception other) {
                throw new IllegalStateException("while trying to parse file " + schemaFile, other);
            }
            // go over avro results and determine what schemas are new in the file parsed
            Map<String, Schema> actuallyNewSchemas = processParseResults(schemaFile, result, successfullyParsed, importableSchemas, externalSchemas);
            // store new schemas
            actuallyNewSchemas.forEach((fullName, schema) -> {
                if (successfullyParsed.put(fullName, SchemaDetails.fromFile(schema, schemaFile, schema == result.getMainSchema())) != null) {
                    throw new IllegalStateException();
                }
            });
        }
        if (!madeProgress) {
            // classify our issues
            failedFiles.forEach((file, issue) -> issue.setClassification(classifyIssue(issue.getException())));
            // terminate for any issues that we cant handle
            throwForFatalErrors(failedFiles, successfullyParsed, importableSchemas);
            // if we got here none of the issues are fatal
            if (allowClasspathLookup) {
                ClasspathFishingResults fishingResults = goFishingOnTheClasspath(failedFiles);
                Map<String, SchemaDetails> loot = fishingResults.getFqcnsFound();
                if (!loot.isEmpty()) {
                    // yay, we live to iterate another day!
                    externalSchemas.putAll(loot);
                } else {
                    throw new IllegalArgumentException("unable to find records " + fishingResults.getFqcnsNotFound() + " used in " + failedFiles.keySet() + ". not even on the classpath");
                }
            } else {
                // TODO - improve on this error msg?
                throw new IllegalArgumentException("cannot make progress. files left: " + failedFiles + ". did not examine classpath");
            }
        }
        toTry = failedFiles.keySet();
    }
    LOG.info("parsed {} schemas out of {} files in {} passes", successfullyParsed.size(), toParse.size(), passNumber);
    return successfullyParsed;
}

Example 2 with SchemaParseResult

use of com.linkedin.avroutil1.compatibility.SchemaParseResult in project avro-util by linkedin.

the class Avro15Adapter method parse.

@Override
public SchemaParseResult parse(String schemaJson, SchemaParseConfiguration desiredConf, Collection<Schema> known) {
    Schema.Parser parser = new Schema.Parser();
    boolean validateNames = true;
    boolean validateDefaults = false;
    if (desiredConf != null) {
        validateNames = desiredConf.validateNames();
        validateDefaults = desiredConf.validateDefaultValues();
    }
    SchemaParseConfiguration configUsed = new SchemaParseConfiguration(validateNames, validateDefaults);
    parser.setValidate(validateNames);
    if (known != null && !known.isEmpty()) {
        Map<String, Schema> knownByFullName = new HashMap<>(known.size());
        for (Schema s : known) {
            knownByFullName.put(s.getFullName(), s);
        }
        parser.addTypes(knownByFullName);
    }
    Schema mainSchema = parser.parse(schemaJson);
    if (validateDefaults) {
        // avro 1.5 doesnt properly validate default values, so we have to do it ourselves
        SchemaValidator validator = new SchemaValidator(configUsed, known);
        // will throw on issues
        AvroSchemaUtil.traverseSchema(mainSchema, validator);
    }
    Map<String, Schema> knownByFullName = parser.getTypes();
    return new SchemaParseResult(mainSchema, knownByFullName, configUsed);
}

Example 3 with SchemaParseResult

use of com.linkedin.avroutil1.compatibility.SchemaParseResult in project avro-util by linkedin.

the class Avro18Adapter method parse.

@Override
public SchemaParseResult parse(String schemaJson, SchemaParseConfiguration desiredConf, Collection<Schema> known) {
    Schema.Parser parser = new Schema.Parser();
    boolean validateNames = true;
    boolean validateDefaults = true;
    if (desiredConf != null) {
        validateNames = desiredConf.validateNames();
        validateDefaults = desiredConf.validateDefaultValues();
    }
    parser.setValidate(validateNames);
    parser.setValidateDefaults(validateDefaults);
    if (known != null && !known.isEmpty()) {
        Map<String, Schema> knownByFullName = new HashMap<>(known.size());
        for (Schema s : known) {
            knownByFullName.put(s.getFullName(), s);
        }
        parser.addTypes(knownByFullName);
    }
    Schema mainSchema = parser.parse(schemaJson);
    // avro 1.8 can (optionally) validate defaults - so even if the user asked for that we have no further work to do
    Map<String, Schema> knownByFullName = parser.getTypes();
    return new SchemaParseResult(mainSchema, knownByFullName, new SchemaParseConfiguration(validateNames, validateDefaults));
}

Example 4 with SchemaParseResult

use of com.linkedin.avroutil1.compatibility.SchemaParseResult in project avro-util by linkedin.

the class Avro16Adapter method parse.

@Override
public SchemaParseResult parse(String schemaJson, SchemaParseConfiguration desiredConf, Collection<Schema> known) {
    Schema.Parser parser = new Schema.Parser();
    boolean validateNames = true;
    boolean validateDefaults = false;
    if (desiredConf != null) {
        validateNames = desiredConf.validateNames();
        validateDefaults = desiredConf.validateDefaultValues();
    }
    SchemaParseConfiguration configUsed = new SchemaParseConfiguration(validateNames, validateDefaults);
    parser.setValidate(validateNames);
    if (known != null && !known.isEmpty()) {
        Map<String, Schema> knownByFullName = new HashMap<>(known.size());
        for (Schema s : known) {
            knownByFullName.put(s.getFullName(), s);
        }
        parser.addTypes(knownByFullName);
    }
    Schema mainSchema = parser.parse(schemaJson);
    if (validateDefaults) {
        // avro 1.6 doesnt properly validate default values, so we have to do it ourselves
        SchemaValidator validator = new SchemaValidator(configUsed, known);
        // will throw on issues
        AvroSchemaUtil.traverseSchema(mainSchema, validator);
    }
    Map<String, Schema> knownByFullName = parser.getTypes();
    return new SchemaParseResult(mainSchema, knownByFullName, configUsed);
}

Example 5 with SchemaParseResult

use of com.linkedin.avroutil1.compatibility.SchemaParseResult in project avro-util by linkedin.

the class Avro19Adapter method parse.

@Override
public SchemaParseResult parse(String schemaJson, SchemaParseConfiguration desiredConf, Collection<Schema> known) {
    Schema.Parser parser = new Schema.Parser();
    boolean validateNames = true;
    boolean validateDefaults = true;
    if (desiredConf != null) {
        validateNames = desiredConf.validateNames();
        validateDefaults = desiredConf.validateDefaultValues();
    }
    parser.setValidate(validateNames);
    parser.setValidateDefaults(validateDefaults);
    if (known != null && !known.isEmpty()) {
        Map<String, Schema> knownByFullName = new HashMap<>(known.size());
        for (Schema s : known) {
            knownByFullName.put(s.getFullName(), s);
        }
        parser.addTypes(knownByFullName);
    }
    Schema mainSchema = parser.parse(schemaJson);
    Map<String, Schema> knownByFullName = parser.getTypes();
    // todo - depending on how https://issues.apache.org/jira/browse/AVRO-2742 is settled, may need to use our own validator here
    return new SchemaParseResult(mainSchema, knownByFullName, new SchemaParseConfiguration(validateNames, validateDefaults));
}