Examples with Tokenizer org.develnext.jphp.core.tokenizer.Tokenizer used on opensource projects

Example 21 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testStringSlashes.

@Test
public void testStringSlashes() throws IOException {
    Token token;
    Tokenizer tokenizer = new Tokenizer(new Context(" 'foo\\'bar' \"foo\\\"bar\""));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals("foo'bar", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals("foo\"bar", ((StringExprToken) token).getValue());
}

Example 22 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testSimple.

@Test
public void testSimple() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context(""));
    assertNull(tokenizer.nextToken());
    assertEquals("", tokenizer.getCode());
    tokenizer = new Tokenizer(new Context(" "));
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("  "));
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("\t"));
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("\n"));
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("\r"));
    assertNull(tokenizer.nextToken());
}

Example 23 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class JvmCompilerCase method getSyntaxTree.

protected List<Token> getSyntaxTree(Context context) {
    Tokenizer tokenizer = null;
    try {
        tokenizer = new Tokenizer(context);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    SyntaxAnalyzer analyzer = new SyntaxAnalyzer(environment, tokenizer);
    return analyzer.getTree();
}

Example 24 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class CommentsTest method testBug154.

@Test
public void testBug154() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("/*// */"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.BLOCK, ((CommentToken) token).getKind());
    assertEquals("// ", ((CommentToken) token).getComment());
}

Example 25 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testMagicString.

@Test
public void testMagicString() throws IOException {
    Token token;
    Tokenizer tokenizer = new Tokenizer(new Context("\"\\.{$foo}\""));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(".{$foo}", ((StringExprToken) token).getValue());
    assertEquals(1, ((StringExprToken) token).getSegments().size());
    StringExprToken.Segment segment = ((StringExprToken) token).getSegments().get(0);
    assertEquals(1, segment.from);
    assertEquals(7, segment.to);
}