Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 86 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method testToStringStartStop2.

@Test
public void testToStringStartStop2() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    String input = "x = 3 * 0 + 2 * 0;";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    String result = tokens.getTokenStream().getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);
    tokens.replace(4, 8, "0");
    stream.fill();
    // replace 3 * 0 with 0
    result = tokens.getText();
    expecting = "x = 0 + 2 * 0;";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(0, 17));
    expecting = "x = 0 + 2 * 0;";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(4, 8));
    expecting = "0";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(0, 8));
    expecting = "x = 0";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(12, 16));
    expecting = "2 * 0";
    assertEquals(expecting, result);
    tokens.insertAfter(17, "// comment");
    result = tokens.getText(Interval.of(12, 18));
    expecting = "2 * 0;// comment";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(0, 8));
    stream.fill();
    // try again after insert at end
    expecting = "x = 0";
    assertEquals(expecting, result);
}

Example 87 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder.

// Test Fix for https://github.com/antlr/antlr4/issues/550
@Test
public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "aa";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(0, "<b>");
    tokens.insertAfter(0, "</b>");
    tokens.insertBefore(1, "<b>");
    tokens.insertAfter(1, "</b>");
    String result = tokens.getText();
    // fails with <b>a<b></b>a</b>"
    String expecting = "<b>a</b><b>a</b>";
    assertEquals(expecting, result);
}

Example 88 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method test2ReplaceMiddleIndex.

@Test
public void test2ReplaceMiddleIndex() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, "x");
    tokens.replace(1, "y");
    String result = tokens.getText();
    String expecting = "ayc";
    assertEquals(expecting, result);
}

Example 89 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method testCombineInsertOnLeftWithDelete.

@Test
public void testCombineInsertOnLeftWithDelete() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.delete(0, 2);
    tokens.insertBefore(0, "z");
    stream.fill();
    // combine with left edge of rewrite
    String result = tokens.getText();
    String expecting = "z";
    stream.fill();
    // make sure combo is not znull
    assertEquals(expecting, result);
}

Example 90 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method testLeaveAloneDisjointInsert2.

@Test
public void testLeaveAloneDisjointInsert2() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 3, "foo");
    tokens.insertBefore(1, "x");
    String result = tokens.getText();
    String expecting = "axbfoo";
    assertEquals(expecting, result);
}