Examples with TokenStream org.antlr.v4.runtime.TokenStream used on opensource projects

Example 11 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testCompleteBufferAfterConsuming.

@Test
public void testCompleteBufferAfterConsuming() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    Token t = tokens.LT(1);
    while (t.getType() != Token.EOF) {
        tokens.consume();
        t = tokens.LT(1);
    }
    String result = tokens.getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);
}

Example 12 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestBufferedTokenStream method test2ndToken.

@Test
public void test2ndToken() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    String result = tokens.LT(2).getText();
    String expecting = " ";
    assertEquals(expecting, result);
}

Example 13 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testLookback.

@Test
public void testLookback() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    // get x into buffer
    tokens.consume();
    Token t = tokens.LT(-1);
    assertEquals("x", t.getText());
    tokens.consume();
    // consume '='
    tokens.consume();
    t = tokens.LT(-3);
    assertEquals("x", t.getText());
    t = tokens.LT(-2);
    assertEquals(" ", t.getText());
    t = tokens.LT(-1);
    assertEquals("=", t.getText());
}

Example 14 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testFirstToken.

@Test
public void testFirstToken() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    String result = tokens.LT(1).getText();
    String expecting = "x";
    assertEquals(expecting, result);
}

Example 15 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestATNParserPrediction method checkPredictedAlt.

/** first check that the ATN predicts right alt.
	 *  Then check adaptive prediction.
	 */
public void checkPredictedAlt(LexerGrammar lg, Grammar g, int decision, String inputString, int expectedAlt) {
    Tool.internalOption_ShowATNConfigsInDFA = true;
    ATN lexatn = createATN(lg, true);
    LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn, new DFA[] { new DFA(lexatn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, new PredictionContextCache());
    IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
    //		System.out.println(types);
    semanticProcess(lg);
    g.importVocab(lg);
    semanticProcess(g);
    ParserATNFactory f = new ParserATNFactory(g);
    ATN atn = f.createATN();
    DOTGenerator dot = new DOTGenerator(g);
    Rule r = g.getRule("a");
    //		if ( r!=null) System.out.println(dot.getDOT(atn.ruleToStartState[r.index]));
    r = g.getRule("b");
    //		if ( r!=null) System.out.println(dot.getDOT(atn.ruleToStartState[r.index]));
    r = g.getRule("e");
    //		if ( r!=null) System.out.println(dot.getDOT(atn.ruleToStartState[r.index]));
    r = g.getRule("ifstat");
    //		if ( r!=null) System.out.println(dot.getDOT(atn.ruleToStartState[r.index]));
    r = g.getRule("block");
    //		if ( r!=null) System.out.println(dot.getDOT(atn.ruleToStartState[r.index]));
    // Check ATN prediction
    //		ParserATNSimulator interp = new ParserATNSimulator(atn);
    TokenStream input = new IntTokenStream(types);
    ParserInterpreterForTesting interp = new ParserInterpreterForTesting(g, input);
    int alt = interp.adaptivePredict(input, decision, ParserRuleContext.EMPTY);
    assertEquals(expectedAlt, alt);
    // Check adaptive prediction
    input.seek(0);
    alt = interp.adaptivePredict(input, decision, null);
    assertEquals(expectedAlt, alt);
    // run 2x; first time creates DFA in atn
    input.seek(0);
    alt = interp.adaptivePredict(input, decision, null);
    assertEquals(expectedAlt, alt);
}