Examples with Lexer org.antlr.v4.runtime.Lexer used on opensource projects

Example 36 with Lexer

use of org.antlr.v4.runtime.Lexer in project ballerina by ballerina-lang.

the class TomlProcessor method parseTomlContent.

/**
 * Generate the proxy object by passing in the toml file.
 *
 * @param stream charstream object containing the content
 * @return proxy object
 */
public static ParseTree parseTomlContent(CharStream stream) {
    TomlLexer lexer = new TomlLexer(stream);
    // Get a list of matched tokens
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // Pass the tokens to the parser
    TomlParser parser = new TomlParser(tokens);
    return parser.toml();
}

Example 37 with Lexer

use of org.antlr.v4.runtime.Lexer in project graphql-java by graphql-java.

the class Parser method parseDocument.

public Document parseDocument(String input) {
    GraphqlLexer lexer = new GraphqlLexer(CharStreams.fromString(input));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    GraphqlParser parser = new GraphqlParser(tokens);
    parser.removeErrorListeners();
    parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
    parser.setErrorHandler(new BailErrorStrategy());
    GraphqlParser.DocumentContext document = parser.document();
    GraphqlAntlrToLanguage antlrToLanguage = new GraphqlAntlrToLanguage(tokens);
    antlrToLanguage.visitDocument(document);
    Token stop = document.getStop();
    List<Token> allTokens = tokens.getTokens();
    if (stop != null && allTokens != null && !allTokens.isEmpty()) {
        Token last = allTokens.get(allTokens.size() - 1);
        // 
        // do we have more tokens in the stream than we consumed in the parse?
        // if yes then its invalid.  We make sure its the same channel
        boolean notEOF = last.getType() != Token.EOF;
        boolean lastGreaterThanDocument = last.getTokenIndex() > stop.getTokenIndex();
        boolean sameChannel = last.getChannel() == stop.getChannel();
        if (notEOF && lastGreaterThanDocument && sameChannel) {
            throw new ParseCancellationException("There are more tokens in the query that have not been consumed");
        }
    }
    return antlrToLanguage.getResult();
}

Example 38 with Lexer

use of org.antlr.v4.runtime.Lexer in project robozonky by RoboZonky.

the class SideEffectFreeParser method modifyInterpreter.

private static void modifyInterpreter(final NaturalLanguageStrategyLexer l) {
    final int originalSize = l.getInterpreter().decisionToDFA.length;
    // give our own array so the static one isn't used
    final DFA[] emptyDFA = new DFA[originalSize];
    final LexerATNSimulator newInterpreter = new LexerATNSimulator(l, l.getATN(), emptyDFA, new PredictionContextCache());
    // initialize our array so that the lexer functions properly
    newInterpreter.clearDFA();
    // replace the interpreter to bypass all static caches
    l.setInterpreter(newInterpreter);
}

Example 39 with Lexer

use of org.antlr.v4.runtime.Lexer in project titan.EclipsePlug-ins by eclipse.

the class CfgParseTreePrinter method printHiddenTokensBefore.

/**
 * Builds hidden tokens before the token
 * @param aToken the token, this will NOT be printed
 * @param aTokens token list from the lexer (all, hidden and not hidden also)
 */
private void printHiddenTokensBefore(final Token aToken, final List<Token> aTokens) {
    final int tokenIndex = aToken.getTokenIndex();
    if (tokenIndex == -1) {
        // because token has no index in the token list.
        return;
    }
    int startHiddenIndex = tokenIndex;
    while (isHiddenToken(startHiddenIndex - 1, aTokens)) {
        startHiddenIndex--;
    }
    for (int i = startHiddenIndex; i < tokenIndex; i++) {
        final Token t = aTokens.get(i);
        final String tokenText = t.getText();
        mSb.append(tokenText != null ? tokenText : "");
    }
}

Example 40 with Lexer

use of org.antlr.v4.runtime.Lexer in project titan.EclipsePlug-ins by eclipse.

the class ASN1ReferenceParser method parseReference.

private Reference parseReference(final IFile file, final String input, final int line, final int offset) {
    Reference reference = null;
    StringReader reader = new StringReader(input);
    CharStream charStream = new UnbufferedCharStream(reader);
    Asn1Lexer lexer = new Asn1Lexer(charStream);
    lexer.setTokenFactory(new TokenWithIndexAndSubTokensFactory(true));
    ASN1Listener lexerListener = new ASN1Listener();
    // remove ConsoleErrorListener
    lexer.removeErrorListeners();
    lexer.addErrorListener(lexerListener);
    ModuleLevelTokenStreamTracker tracker = new ModuleLevelTokenStreamTracker(lexer);
    tracker.discard(Asn1Lexer.WS);
    tracker.discard(Asn1Lexer.MULTILINECOMMENT);
    tracker.discard(Asn1Lexer.SINGLELINECOMMENT);
    Asn1Parser parser = new Asn1Parser(tracker);
    parser.setProject(file.getProject());
    parser.setActualFile(file);
    parser.setLine(line);
    parser.setOffset(offset);
    parser.setBuildParseTree(false);
    ASN1Listener parserListener = new ASN1Listener();
    // remove ConsoleErrorListener
    parser.removeErrorListeners();
    parser.addErrorListener(parserListener);
    reference = parser.pr_parseReference().reference;
    return reference;
}