Examples with Token net.morimekta.providence.serializer.pretty.Token used on opensource projects

Example 1 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ProvidenceConfigParser method parseConfigRecursively.

@SuppressWarnings("unchecked")
<M extends PMessage<M, F>, F extends PField> Pair<M, Set<String>> parseConfigRecursively(@Nonnull Path file, M parent, String[] stack) throws IOException {
    Tokenizer tokenizer;
    try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file.toFile()))) {
        // Non-enclosed content, meaning we should read the whole file immediately.
        tokenizer = new Tokenizer(new Utf8StreamReader(in), Tokenizer.DEFAULT_BUFFER_SIZE, true);
    }
    ProvidenceConfigContext context = new ProvidenceConfigContext();
    Set<String> includedFilePaths = new TreeSet<>();
    includedFilePaths.add(canonicalFileLocation(file).toString());
    Stage lastStage = Stage.INCLUDES;
    M result = null;
    Token token = tokenizer.peek();
    while (token != null) {
        tokenizer.next();
        if (lastStage == Stage.MESSAGE) {
            throw new TokenizerException(token, "Unexpected token '" + token.asString() + "', expected end of file.").setLine(tokenizer.getLine());
        } else if (INCLUDE.equals(token.asString())) {
            // if include && stage == INCLUDES --> INCLUDES
            if (lastStage != Stage.INCLUDES) {
                throw new TokenizerException(token, "Include added after defines or message. Only one def block allowed.").setLine(tokenizer.getLine());
            }
            token = tokenizer.expectLiteral("file to be included");
            String includedFilePath = token.decodeLiteral(strict);
            PMessage included;
            Path includedFile;
            try {
                includedFile = resolveFile(file, includedFilePath);
                Pair<PMessage, Set<String>> tmp = checkAndParseInternal(includedFile, null, stack);
                if (tmp != null) {
                    includedFilePaths.add(includedFile.toString());
                    includedFilePaths.addAll(tmp.second);
                    included = tmp.first;
                } else {
                    included = null;
                }
            } catch (FileNotFoundException e) {
                throw new TokenizerException(token, "Included file \"%s\" not found.", includedFilePath).setLine(tokenizer.getLine());
            }
            token = tokenizer.expectIdentifier("the token 'as'");
            if (!AS.equals(token.asString())) {
                throw new TokenizerException(token, "Expected token 'as' after included file \"%s\".", includedFilePath).setLine(tokenizer.getLine());
            }
            token = tokenizer.expectIdentifier("Include alias");
            String alias = token.asString();
            if (RESERVED_WORDS.contains(alias)) {
                throw new TokenizerException(token, "Alias \"%s\" is a reserved word.", alias).setLine(tokenizer.getLine());
            }
            if (context.containsReference(alias)) {
                throw new TokenizerException(token, "Alias \"%s\" is already used.", alias).setLine(tokenizer.getLine());
            }
            context.setInclude(alias, included);
        } else if (DEF.equals(token.asString())) {
            // if params && stage == DEF --> DEF
            lastStage = Stage.DEFINES;
            parseDefinitions(context, tokenizer);
        } else if (token.isQualifiedIdentifier()) {
            // if a.b (type identifier) --> MESSAGE
            lastStage = Stage.MESSAGE;
            PMessageDescriptor<M, F> descriptor;
            try {
                descriptor = (PMessageDescriptor) registry.getDeclaredType(token.asString());
            } catch (IllegalArgumentException e) {
                // even in non-strict mode.
                if (strict || stack.length == 1) {
                    throw new TokenizerException(token, "Unknown declared type: %s", token.asString()).setLine(tokenizer.getLine());
                }
                return null;
            }
            result = parseConfigMessage(tokenizer, context, descriptor.builder(), parent, file);
        } else {
            throw new TokenizerException(token, "Unexpected token '" + token.asString() + "'. Expected include, defines or message type").setLine(tokenizer.getLine());
        }
        token = tokenizer.peek();
    }
    if (result == null) {
        throw new TokenizerException("No message in config: " + file.getFileName().toString());
    }
    return Pair.create(result, includedFilePaths);
}

Example 2 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ProvidenceConfigParser method parseDefinitions.

@SuppressWarnings("unchecked")
void parseDefinitions(ProvidenceConfigContext context, Tokenizer tokenizer) throws IOException {
    Token token = tokenizer.expect("defines group start or identifier");
    if (token.isIdentifier()) {
        String name = context.initReference(token, tokenizer);
        tokenizer.expectSymbol("def value sep", Token.kFieldValueSep);
        context.setReference(name, parseDefinitionValue(context, tokenizer));
    } else if (token.isSymbol(Token.kMessageStart)) {
        token = tokenizer.expect("define or end");
        while (!token.isSymbol(Token.kMessageEnd)) {
            if (!token.isIdentifier()) {
                throw new TokenizerException(token, "Token '%s' is not valid reference name.", token.asString()).setLine(tokenizer.getLine());
            }
            String name = context.initReference(token, tokenizer);
            tokenizer.expectSymbol("def value sep", Token.kFieldValueSep);
            context.setReference(name, parseDefinitionValue(context, tokenizer));
            token = tokenizer.expect("next define or end");
        }
    } else {
        throw new TokenizerException(token, "Unexpected token after def: '%s'", token.asString()).setLine(tokenizer.getLine());
    }
}

Example 3 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ProvidenceConfigParser method parseMessage.

@SuppressWarnings("unchecked")
<M extends PMessage<M, F>, F extends PField> M parseMessage(@Nonnull Tokenizer tokenizer, @Nonnull ProvidenceConfigContext context, @Nonnull PMessageBuilder<M, F> builder) throws IOException {
    PMessageDescriptor<M, F> descriptor = builder.descriptor();
    Token token = tokenizer.expect("object end or field");
    while (!token.isSymbol(Token.kMessageEnd)) {
        if (!token.isIdentifier()) {
            throw new TokenizerException(token, "Invalid field name: " + token.asString()).setLine(tokenizer.getLine());
        }
        F field = descriptor.findFieldByName(token.asString());
        if (field == null) {
            if (strict) {
                throw new TokenizerException("No such field " + token.asString() + " in " + descriptor.getQualifiedName()).setLine(tokenizer.getLine());
            } else {
                token = tokenizer.expect("field value sep, message start or reference start");
                if (token.isSymbol(DEFINE_REFERENCE)) {
                    context.setReference(context.initReference(tokenizer.expectIdentifier("reference name"), tokenizer), null);
                    // Ignore reference.
                    token = tokenizer.expect("field value sep or message start");
                }
                if (token.isSymbol(Token.kFieldValueSep)) {
                    token = tokenizer.expect("value declaration");
                } else if (!token.isSymbol(Token.kMessageStart)) {
                    throw new TokenizerException(token, "Expected field-value separator or inherited message").setLine(tokenizer.getLine());
                }
                // Non-strict will just consume unknown fields, this way
                // we can be forward-compatible when reading config.
                consumeValue(context, tokenizer, token);
                token = nextNotLineSep(tokenizer, "field or message end");
                continue;
            }
        }
        if (field.getType() == PType.MESSAGE) {
            // go recursive with optional
            String reference = null;
            char symbol = tokenizer.expectSymbol("Message assigner or start", Token.kFieldValueSep, Token.kMessageStart, DEFINE_REFERENCE);
            if (symbol == DEFINE_REFERENCE) {
                Token ref = tokenizer.expectIdentifier("reference name");
                if (strict) {
                    throw tokenizer.failure(ref, "Reusable objects are not allowed in strict mode.");
                }
                reference = context.initReference(ref, tokenizer);
                symbol = tokenizer.expectSymbol("Message assigner or start after " + reference, Token.kFieldValueSep, Token.kMessageStart);
            }
            PMessageBuilder bld;
            if (symbol == Token.kFieldValueSep) {
                token = tokenizer.expect("reference or message start");
                if (UNDEFINED.equals(token.asString())) {
                    // unset.
                    builder.clear(field.getId());
                    context.setReference(reference, null);
                    // special casing this, as we don't want to duplicate the parse line below.
                    token = nextNotLineSep(tokenizer, "field or message end");
                    continue;
                }
                // overwrite with new.
                bld = ((PMessageDescriptor) field.getDescriptor()).builder();
                if (token.isReferenceIdentifier()) {
                    // Inherit from reference.
                    try {
                        PMessage ref = resolve(context, token, tokenizer, field.getDescriptor());
                        if (ref != null) {
                            bld.merge(ref);
                        } else {
                            if (tokenizer.peek().isSymbol(Token.kMessageStart)) {
                                throw new TokenizerException(token, "Inherit from unknown reference %s", token.asString()).setLine(tokenizer.getLine());
                            } else if (strict) {
                                throw new TokenizerException(token, "Unknown reference %s", token.asString()).setLine(tokenizer.getLine());
                            }
                        }
                    } catch (ProvidenceConfigException e) {
                        throw new TokenizerException(token, "Unknown inherited reference '%s'", token.asString()).setLine(tokenizer.getLine());
                    }
                    token = tokenizer.expect("after message reference");
                    // we assume a new field or end of current message.
                    if (!token.isSymbol(Token.kMessageStart)) {
                        builder.set(field.getId(), context.setReference(reference, bld.build()));
                        continue;
                    }
                } else if (!token.isSymbol(Token.kMessageStart)) {
                    throw new TokenizerException(token, "Unexpected token " + token.asString() + ", expected message start").setLine(tokenizer.getLine());
                }
            } else {
                // extend in-line.
                bld = builder.mutator(field.getId());
            }
            builder.set(field.getId(), context.setReference(reference, parseMessage(tokenizer, context, bld)));
        } else if (field.getType() == PType.MAP) {
            // maps can be extended the same way as
            token = tokenizer.expect("field sep or value start");
            Map baseValue = new LinkedHashMap<>();
            String reference = null;
            if (token.isSymbol(DEFINE_REFERENCE)) {
                Token ref = tokenizer.expectIdentifier("reference name");
                if (strict) {
                    throw tokenizer.failure(ref, "Reusable objects are not allowed in strict mode.");
                }
                reference = context.initReference(ref, tokenizer);
                token = tokenizer.expect("field sep or value start");
            }
            if (token.isSymbol(Token.kFieldValueSep)) {
                token = tokenizer.expect("field id or start");
                if (UNDEFINED.equals(token.asString())) {
                    builder.clear(field.getId());
                    context.setReference(reference, null);
                    token = tokenizer.expect("message end or field");
                    continue;
                } else if (token.isReferenceIdentifier()) {
                    try {
                        baseValue = resolve(context, token, tokenizer, field.getDescriptor());
                    } catch (ProvidenceConfigException e) {
                        throw new TokenizerException(token, e.getMessage()).setLine(tokenizer.getLine());
                    }
                    token = tokenizer.expect("map start or next field");
                    if (!token.isSymbol(Token.kMessageStart)) {
                        builder.set(field.getId(), context.setReference(reference, baseValue));
                        continue;
                    } else if (baseValue == null) {
                        baseValue = new LinkedHashMap<>();
                    }
                }
            } else {
                baseValue.putAll(builder.build().get(field.getId()));
            }
            if (!token.isSymbol(Token.kMessageStart)) {
                throw new TokenizerException(token, "Expected map start, but got '%s'", token.asString()).setLine(tokenizer.getLine());
            }
            Map map = parseMapValue(tokenizer, context, (PMap) field.getDescriptor(), baseValue);
            builder.set(field.getId(), context.setReference(reference, map));
        } else {
            String reference = null;
            // Simple fields *must* have the '=' separation, may have '&' reference.
            if (tokenizer.expectSymbol("field value sep", Token.kFieldValueSep, DEFINE_REFERENCE) == DEFINE_REFERENCE) {
                Token ref = tokenizer.expectIdentifier("reference name");
                if (strict) {
                    throw tokenizer.failure(ref, "Reusable objects are not allowed in strict mode.");
                }
                reference = context.initReference(ref, tokenizer);
                tokenizer.expectSymbol("field value sep", Token.kFieldValueSep);
            }
            token = tokenizer.expect("field value");
            if (UNDEFINED.equals(token.asString())) {
                builder.clear(field.getId());
                context.setReference(reference, null);
            } else {
                Object value = parseFieldValue(token, tokenizer, context, field.getDescriptor(), strict);
                builder.set(field.getId(), context.setReference(reference, value));
            }
        }
        token = nextNotLineSep(tokenizer, "field or message end");
    }
    return builder.build();
}

Example 4 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ProvidenceConfigUtil method consumeValue.

static void consumeValue(@Nonnull ProvidenceConfigContext context, @Nonnull Tokenizer tokenizer, @Nonnull Token token) throws IOException {
    boolean isMessage = false;
    if (UNDEFINED.equals(token.asString())) {
        // ignore undefined.
        return;
    } else if (token.asString().equals(Token.B64)) {
        tokenizer.expectSymbol("b64 body start", Token.kParamsStart);
        tokenizer.readBinary(Token.kParamsEnd);
    } else if (token.asString().equals(Token.HEX)) {
        tokenizer.expectSymbol("hex body start", Token.kParamsStart);
        tokenizer.readBinary(Token.kParamsEnd);
    } else if (token.isReferenceIdentifier()) {
        if (!tokenizer.peek("message start").isSymbol(Token.kMessageStart)) {
            // just a reference.
            return;
        }
        // reference + message.
        isMessage = true;
        token = tokenizer.expect("start of message");
    }
    if (token.isSymbol(Token.kMessageStart)) {
        // message or map.
        token = tokenizer.expect("map or message first entry");
        if (token.isSymbol(Token.kMessageEnd)) {
            return;
        }
        Token firstSep = tokenizer.peek("First separator");
        if (!isMessage && !firstSep.isSymbol(Token.kFieldValueSep) && !firstSep.isSymbol(Token.kMessageStart) && !firstSep.isSymbol(DEFINE_REFERENCE)) {
            // assume map.
            while (!token.isSymbol(Token.kMessageEnd)) {
                if (!token.isIdentifier() && token.isReferenceIdentifier()) {
                    throw new TokenizerException(token, "Invalid map key: " + token.asString()).setLine(tokenizer.getLine());
                }
                consumeValue(context, tokenizer, token);
                tokenizer.expectSymbol("key value sep.", Token.kKeyValueSep);
                consumeValue(context, tokenizer, tokenizer.expect("map value"));
                // maps do *not* require separator, but allows ',' separator, and separator after last.
                token = nextNotLineSep(tokenizer, "map key, sep or end");
            }
        } else {
            // assume message.
            while (!token.isSymbol(Token.kMessageEnd)) {
                if (!token.isIdentifier()) {
                    throw new TokenizerException(token, "Invalid field name: " + token.asString()).setLine(tokenizer.getLine());
                }
                token = tokenizer.expect("field value sep");
                if (token.isSymbol(DEFINE_REFERENCE)) {
                    token = tokenizer.expectIdentifier("reference name");
                    context.setReference(context.initReference(token, tokenizer), null);
                    token = tokenizer.expect("field value sep");
                }
                if (token.isSymbol(Token.kMessageStart)) {
                    // direct inheritance of message field.
                    consumeValue(context, tokenizer, token);
                } else if (token.isSymbol(Token.kFieldValueSep)) {
                    consumeValue(context, tokenizer, tokenizer.expect("field value"));
                } else {
                    throw new TokenizerException(token, "Unknown field value sep: " + token.asString()).setLine(tokenizer.getLine());
                }
                token = nextNotLineSep(tokenizer, "message field or end");
            }
        }
    } else if (token.isSymbol(Token.kListStart)) {
        token = tokenizer.expect("list value or end");
        while (!token.isSymbol(Token.kListEnd)) {
            consumeValue(context, tokenizer, token);
            // lists and sets require list separator (,), and allows trailing separator.
            if (tokenizer.expectSymbol("list separator or end", Token.kLineSep1, Token.kListEnd) == Token.kListEnd) {
                break;
            }
            token = tokenizer.expect("list value or end");
        }
    }
}

Example 5 with Token

use of net.morimekta.providence.serializer.pretty.Token in project providence by morimekta.

the class ThriftProgramParser method parseMessage.

private MessageType parseMessage(ThriftTokenizer tokenizer, String variant, String comment, Set<String> includedPrograms) throws IOException {
    MessageType._Builder struct = MessageType.builder();
    if (comment != null) {
        struct.setDocumentation(comment);
        comment = null;
    }
    boolean union = variant.equals("union");
    if (!variant.equals("struct")) {
        struct.setVariant(MessageVariant.valueForName(variant.toUpperCase(Locale.US)));
    }
    Token nameToken = tokenizer.expectIdentifier("message name identifier");
    String name = nameToken.asString();
    if (!allowedNameIdentifier(name)) {
        throw tokenizer.failure(nameToken, "Message with reserved name: " + name);
    }
    struct.setName(name);
    int nextAutoFieldKey = -1;
    tokenizer.expectSymbol("message start", Token.kMessageStart);
    Set<String> fieldNames = new HashSet<>();
    Set<String> fieldNameVariants = new HashSet<>();
    Set<Integer> fieldIds = new HashSet<>();
    while (true) {
        Token token = tokenizer.expect("field def or message end");
        if (token.isSymbol(Token.kMessageEnd)) {
            break;
        } else if (token.strEquals(kLineCommentStart)) {
            comment = parseDocLine(tokenizer, comment);
            continue;
        } else if (token.strEquals(kBlockCommentStart)) {
            comment = tokenizer.parseDocBlock();
            continue;
        }
        FieldType._Builder field = FieldType.builder();
        field.setDocumentation(comment);
        comment = null;
        if (token.isInteger()) {
            int fId = (int) token.parseInteger();
            if (fId < 1) {
                throw tokenizer.failure(token, "Negative or 0 field id " + fId + " not allowed.");
            }
            if (fieldIds.contains(fId)) {
                throw tokenizer.failure(token, "Field id " + fId + " already exists in " + struct.build().getName());
            }
            fieldIds.add(fId);
            field.setId(fId);
            tokenizer.expectSymbol("field id sep", Token.kKeyValueSep);
            token = tokenizer.expect("field requirement or type", t -> t.isIdentifier() || t.isQualifiedIdentifier());
        } else {
            if (requireFieldId) {
                throw tokenizer.failure(token, "Missing field ID in strict declaration");
            }
            field.setId(nextAutoFieldKey--);
        }
        if (token.strEquals(kRequired)) {
            if (union) {
                throw tokenizer.failure(token, "Found required field in union");
            }
            field.setRequirement(FieldRequirement.REQUIRED);
            token = tokenizer.expect("field type", t -> t.isIdentifier() || t.isQualifiedIdentifier());
        } else if (token.strEquals(kOptional)) {
            if (!union) {
                // All union fields are optional regardless.
                field.setRequirement(FieldRequirement.OPTIONAL);
            }
            token = tokenizer.expect("field type", t -> t.isIdentifier() || t.isQualifiedIdentifier());
        }
        // Get type.... This is mandatory.
        field.setType(parseType(tokenizer, token, includedPrograms));
        nameToken = tokenizer.expectIdentifier("field name");
        String fName = nameToken.asString();
        if (!allowedNameIdentifier(fName)) {
            throw tokenizer.failure(nameToken, "Field with reserved name: " + fName);
        }
        if (fieldNames.contains(fName)) {
            throw tokenizer.failure(nameToken, "Field %s already exists in %s", fName, struct.build().getName());
        }
        if (fieldNameVariants.contains(Strings.camelCase("get", fName))) {
            throw tokenizer.failure(nameToken, "Field %s has field with conflicting name in %s", fName, struct.build().getName());
        }
        fieldNames.add(fName);
        fieldNameVariants.add(Strings.camelCase("get", fName));
        field.setName(fName);
        token = tokenizer.peek("default sep, annotation, field def or message end");
        // Default value
        if (token.isSymbol(Token.kFieldValueSep)) {
            tokenizer.next();
            Token defaultValue = tokenizer.parseValue();
            field.setDefaultValue(defaultValue.asString());
            field.setStartLineNo(defaultValue.getLineNo());
            field.setStartLinePos(defaultValue.getLinePos());
            token = tokenizer.peek("field annotation, def or message end");
        }
        // Annotation
        if (token.isSymbol(Token.kParamsStart)) {
            tokenizer.next();
            field.setAnnotations(parseAnnotations(tokenizer, "field"));
            token = tokenizer.peek("field def or message end");
        }
        struct.addToFields(field.build());
        if (token.isSymbol(Token.kLineSep1) || token.isSymbol(Token.kLineSep2)) {
            tokenizer.next();
        }
    }
    if (tokenizer.hasNext()) {
        Token token = tokenizer.peek("optional annotations");
        if (token.isSymbol(Token.kParamsStart)) {
            tokenizer.next();
            struct.setAnnotations(parseAnnotations(tokenizer, "message"));
        }
    }
    return struct.build();
}