Examples with LexerInterpreter org.antlr.v4.runtime.LexerInterpreter used on opensource projects

Example 21 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testReplaceSingleMiddleThenOverlappingSuperset.

@Test
public void testReplaceSingleMiddleThenOverlappingSuperset() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abcba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 2, "xyz");
    tokens.replace(0, 3, "foo");
    String result = tokens.getText();
    String expecting = "fooa";
    assertEquals(expecting, result);
}

Example 22 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testReplaceAll.

@Test
public void testReplaceAll() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(0, 6, "x");
    String result = tokens.getText();
    String expecting = "x";
    assertEquals(expecting, result);
}

Example 23 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testReplaceThenReplaceSuperset.

@Test
public void testReplaceThenReplaceSuperset() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "xyz");
    tokens.replace(3, 5, "foo");
    stream.fill();
    // overlaps, error
    Exception exc = null;
    try {
        tokens.getText();
    } catch (IllegalArgumentException iae) {
        exc = iae;
    }
    String expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<3>,1:3]..[@5,5:5='b',<2>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<3>,1:2]..[@4,4:4='c',<3>,1:4]:\"xyz\">";
    assertNotNull(exc);
    assertEquals(expecting, exc.getMessage());
}

Example 24 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testToStringStartStop.

@Test
public void testToStringStartStop() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "MUL : '*';\n" + "ASSIGN : '=';\n" + "WS : ' '+;\n");
    // Tokens: 0123456789
    // Input:  x = 3 * 0;
    String input = "x = 3 * 0;";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(4, 8, "0");
    stream.fill();
    // replace 3 * 0 with 0
    String result = tokens.getTokenStream().getText();
    String expecting = "x = 3 * 0;";
    assertEquals(expecting, result);
    result = tokens.getText();
    expecting = "x = 0;";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(0, 9));
    expecting = "x = 0;";
    assertEquals(expecting, result);
    result = tokens.getText(Interval.of(4, 8));
    expecting = "0";
    assertEquals(expecting, result);
}

Example 25 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method test2ReplaceMiddleIndex1InsertBefore.

@Test
public void test2ReplaceMiddleIndex1InsertBefore() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(0, "_");
    tokens.replace(1, "x");
    tokens.replace(1, "y");
    String result = tokens.getText();
    String expecting = "_ayc";
    assertEquals(expecting, result);
}