Examples with Token de.be4.eventbalg.core.parser.node.Token used on opensource projects

Example 31 with Token

use of de.be4.eventbalg.core.parser.node.Token in project probparsers by bendisposto.

the class PreParser method evaluateDefinitionFiles.

private void evaluateDefinitionFiles(final List<Token> list) throws PreParseException, BException, BCompoundException {
    IDefinitionFileProvider cache = null;
    if (contentProvider instanceof IDefinitionFileProvider) {
        cache = (IDefinitionFileProvider) contentProvider;
    }
    for (final Token fileNameToken : list) {
        final List<String> newDoneList = new ArrayList<String>(doneDefFiles);
        try {
            final String fileName = fileNameToken.getText();
            if (doneDefFiles.contains(fileName)) {
                StringBuilder sb = new StringBuilder();
                for (String string : doneDefFiles) {
                    sb.append(string).append(" -> ");
                }
                sb.append(fileName);
                throw new PreParseException(fileNameToken, "Cyclic references in definition files: " + sb.toString());
            }
            IDefinitions definitions;
            if (cache != null && cache.getDefinitions(fileName) != null) {
                definitions = cache.getDefinitions(fileName);
            } else {
                final String content = contentProvider.getFileContent(directory, fileName);
                newDoneList.add(fileName);
                final File file = contentProvider.getFile(directory, fileName);
                String filePath = fileName;
                if (file != null) {
                    filePath = file.getCanonicalPath();
                }
                final BParser parser = new BParser(filePath, parseOptions);
                parser.setDirectory(directory);
                parser.setDoneDefFiles(newDoneList);
                parser.setDefinitions(new Definitions(file));
                parser.parse(content, debugOutput, contentProvider);
                definitions = parser.getDefinitions();
                if (cache != null) {
                    cache.storeDefinition(fileName, definitions);
                }
            }
            defFileDefinitions.addDefinitions(definitions);
            definitionTypes.addAll(definitions.getTypes());
        } catch (final IOException e) {
            throw new PreParseException(fileNameToken, "Definition file cannot be read: " + e.getLocalizedMessage());
        } finally {
        }
    }
}

Example 32 with Token

use of de.be4.eventbalg.core.parser.node.Token in project probparsers by bendisposto.

the class PreParser method evaluateTypes.

private void evaluateTypes(final List<Token> sortedDefinitionList, final Map<Token, Token> definitions) throws PreParseException {
    // use linked list as we rely on pop() and push()
    final LinkedList<Token> remainingDefinitions = new LinkedList<>(sortedDefinitionList);
    final LinkedList<Token> currentlyUnparseableDefinitions = new LinkedList<Token>();
    Set<String> todoDefs = new HashSet<String>();
    for (Token token : remainingDefinitions) {
        todoDefs.add(token.getText());
    }
    // use main parser for the rhs of each definition to determine type
    // if a definition can not be typed this way, it may be due to another
    // definition that is not yet parser (because it appears later in the
    // source code)
    // in this case, the definition is appended to the list again
    // the algorithm terminates if the queue is empty or if no definition
    // has been parsed
    boolean oneParsed = true;
    while (oneParsed) {
        oneParsed = false;
        while (!remainingDefinitions.isEmpty()) {
            final Token definition = remainingDefinitions.pop();
            final Token defRhs = definitions.get(definition);
            Definitions.Type type = null;
            DefinitionType definitionType = determineType(definition, defRhs, todoDefs);
            type = definitionType.type;
            if (type != null) {
                todoDefs.remove(definition.getText());
                oneParsed = true;
                definitionTypes.addTyping(definition.getText(), type);
            // types.addTyping(definition.getText(), type);
            } else {
                currentlyUnparseableDefinitions.push(definition);
            }
        }
        remainingDefinitions.addAll(currentlyUnparseableDefinitions);
        currentlyUnparseableDefinitions.clear();
    }
    if (!remainingDefinitions.isEmpty()) {
        final Token definition = remainingDefinitions.pop();
        final Token defRhs = definitions.get(definition);
        DefinitionType definitionType = determineType(definition, defRhs, todoDefs);
        if (definitionType.errorMessage != null) {
            throw new PreParseException(definitionType.errorMessage + " in file: " + modelFileName);
        // throw new BParseException(definitionType.errorToken,
        // definitionType.errorMessage + " in file: "
        // + modelFileName);
        } else {
            // fall back message
            throw new PreParseException(definition, "[" + definition.getLine() + "," + definition.getPos() + "] expecting wellformed expression, predicate or substitution as DEFINITION body (DEFINITION arguments assumed to be expressions)");
        }
    }
}

Example 33 with Token

use of de.be4.eventbalg.core.parser.node.Token in project probparsers by bendisposto.

the class PreParser method determineDependencies.

private Map<String, Set<String>> determineDependencies(Set<String> definitionNames, Map<Token, Token> definitions) throws PreParseException {
    HashMap<String, Set<String>> dependencies = new HashMap<>();
    for (Entry<Token, Token> entry : definitions.entrySet()) {
        Token nameToken = entry.getKey();
        Token rhsToken = entry.getValue();
        // The FORMULA_PREFIX is needed to switch the lexer state from
        // section to normal. Note, that we do not parse the right hand side
        // of the definition here. Hence FORMULA_PREFIX has no further
        // meaning and substitutions can also be handled by the lexer.
        final Reader reader = new StringReader(BParser.FORMULA_PREFIX + "\n" + rhsToken.getText());
        final BLexer lexer = new BLexer(new PushbackReader(reader, BLexer.PUSHBACK_BUFFER_SIZE), new DefinitionTypes());
        lexer.setParseOptions(parseOptions);
        Set<String> set = new HashSet<>();
        de.be4.classicalb.core.parser.node.Token next = null;
        try {
            next = lexer.next();
            while (!(next instanceof EOF)) {
                if (next instanceof TIdentifierLiteral) {
                    TIdentifierLiteral id = (TIdentifierLiteral) next;
                    String name = id.getText();
                    if (definitionNames.contains(name)) {
                        set.add(name);
                    }
                }
                next = lexer.next();
            }
        } catch (IOException e) {
        } catch (BLexerException e) {
            de.be4.classicalb.core.parser.node.Token errorToken = e.getLastToken();
            final String newMessage = determineNewErrorMessageWithCorrectedPositionInformations(nameToken, rhsToken, errorToken, e.getMessage());
            throw new PreParseException(newMessage);
        } catch (de.be4.classicalb.core.parser.lexer.LexerException e) {
            final String newMessage = determineNewErrorMessageWithCorrectedPositionInformationsWithoutToken(nameToken, rhsToken, e.getMessage());
            throw new PreParseException(newMessage);
        }
        dependencies.put(nameToken.getText(), set);
    }
    return dependencies;
}

Example 34 with Token

use of de.be4.eventbalg.core.parser.node.Token in project probparsers by bendisposto.

the class PreParser method sortDefinitionsByPosition.

@SuppressWarnings("unused")
private LinkedList<Token> sortDefinitionsByPosition(final Map<Token, Token> definitions) {
    // LinkedList will be used as a queue later on!
    // however, the list is needed for collections.sort
    // we can not use a priority queue to sort, as the sorting is done once
    // afterwards, it has to remain unsorted
    final LinkedList<Token> list = new LinkedList<Token>();
    for (final Iterator<Token> iterator = definitions.keySet().iterator(); iterator.hasNext(); ) {
        final Token definition = iterator.next();
        list.add(definition);
    }
    /*
		 * Sort the definitions in order of their appearance in the sourcecode.
		 * Dependencies in between definitions are handled later when computing
		 * there type
		 */
    Collections.sort(list, new Comparator<Token>() {

        public int compare(final Token o1, final Token o2) {
            if (o1.getLine() == o2.getLine()) {
                if (o1.getPos() == o2.getPos())
                    return 0;
                else
                    return o1.getPos() - o2.getPos();
            } else
                return o1.getLine() - o2.getLine();
        }
    });
    return list;
}

Example 35 with Token

use of de.be4.eventbalg.core.parser.node.Token in project probparsers by bendisposto.

the class PrologExceptionPrinter method printBParseException.

private static void printBParseException(final IPrologTermOutput pto, final BParseException e, final String filename, final boolean useIndentation, final boolean lineOneOff) {
    final Token token = e.getToken();
    final SourcePosition pos = token == null ? null : new SourcePosition(token.getLine(), token.getPos());
    printExceptionWithSourcePosition(pto, e, filename, pos, useIndentation, lineOneOff);
}