Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 96 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 97 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}

Example 98 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project L42 by ElvisResearchGroup.

the class Parser method getParser.

public static antlrGenerated.L42Parser getParser(String s) {
    ANTLRInputStream in = new ANTLRInputStream(s);
    antlrGenerated.L42Lexer l = new antlrGenerated.L42Lexer(in);
    /*{
      @Override public void recover(LexerNoViableAltException e) {
        throw new IllegalArgumentException(e); // Bail out
      }
    @Override public void recover(RecognitionException re) {
        throw new IllegalArgumentException(re); // Bail out
      }};*/
    CommonTokenStream t = new CommonTokenStream(l);
    antlrGenerated.L42Parser p = new antlrGenerated.L42Parser(t);
    //    p.setErrorHandler(new BailErrorStrategy());
    return p;
}

Example 99 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestRig method process.

protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, CharStream input) throws IOException, IllegalAccessException, InvocationTargetException, PrintException {
    lexer.setInputStream(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    if (showTokens) {
        for (Token tok : tokens.getTokens()) {
            if (tok instanceof CommonToken) {
                System.out.println(((CommonToken) tok).toString(lexer));
            } else {
                System.out.println(tok.toString());
            }
        }
    }
    if (startRuleName.equals(LEXER_START_RULE_NAME))
        return;
    if (diagnostics) {
        parser.addErrorListener(new DiagnosticErrorListener());
        parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
    }
    if (printTree || gui || psFile != null) {
        parser.setBuildParseTree(true);
    }
    if (SLL) {
        // overrides diagnostics
        parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
    }
    parser.setTokenStream(tokens);
    parser.setTrace(trace);
    try {
        Method startRule = parserClass.getMethod(startRuleName);
        ParserRuleContext tree = (ParserRuleContext) startRule.invoke(parser, (Object[]) null);
        if (printTree) {
            System.out.println(tree.toStringTree(parser));
        }
        if (gui) {
            Trees.inspect(tree, parser);
        }
        if (psFile != null) {
            // Generate postscript
            Trees.save(tree, parser, psFile);
        }
    } catch (NoSuchMethodException nsme) {
        System.err.println("No method for rule " + startRuleName + " or it has arguments");
    }
}

Example 100 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestUnicodeGrammar method parseTreeForGrammarWithInput.

private static String parseTreeForGrammarWithInput(String grammarText, String rootRule, String inputText) throws Exception {
    Grammar grammar = new Grammar(grammarText);
    LexerInterpreter lexEngine = grammar.createLexerInterpreter(CharStreams.fromString(inputText));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens);
    ParseTree parseTree = parser.parse(grammar.rules.get(rootRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(grammar.getRuleNames());
    return Trees.toStringTree(parseTree, nodeTextProvider);
}