Examples with Tokenizer org.develnext.jphp.core.tokenizer.Tokenizer used on opensource projects

Example 11 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testBraces.

@Test
public void testBraces() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context(" :: ->foobar('a', 1, 3.0);"));
    assertTrue(tokenizer.nextToken() instanceof StaticAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof DynamicAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof DoubleExprToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
}

Example 12 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testBug122.

@Test
public void testBug122() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("<<<ESC\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "<?php\n" + "ESC;\n"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals("\n\n\n\n\n<?php", ((StringExprToken) token).getValue());
}

Example 13 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testSimpleOperators.

@Test
public void testSimpleOperators() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("= + - / * % . and or new && || ! xor **"));
    assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
    assertTrue(tokenizer.nextToken() instanceof PlusExprToken);
    assertTrue(tokenizer.nextToken() instanceof MinusExprToken);
    assertTrue(tokenizer.nextToken() instanceof DivExprToken);
    assertTrue(tokenizer.nextToken() instanceof MulExprToken);
    assertTrue(tokenizer.nextToken() instanceof ModExprToken);
    assertTrue(tokenizer.nextToken() instanceof ConcatExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanAnd2ExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanOr2ExprToken);
    assertTrue(tokenizer.nextToken() instanceof NewExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanAndExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanOrExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanNotExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanXorExprToken);
    assertTrue(tokenizer.nextToken() instanceof PowExprToken);
}

Example 14 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testHeredoc.

@Test
public void testHeredoc() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("<<<DOC\n <foobar> \nDOC;\n"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" <foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<\"DOC\"\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<'DOC'\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \\n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("a(<<<DOC\n<foobar>\nDOC\n)\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof BraceExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals("<foobar>", ((StringExprToken) token).getValue());
}

Example 15 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testMacro.

@Test
public void testMacro() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("__LINE__ __FILE__ __DIR__ __METHOD__ __FUNCTION__ __CLASS__ __NAMESPACE__ __TRAIT__"));
    assertTrue(tokenizer.nextToken() instanceof LineMacroToken);
    assertTrue(tokenizer.nextToken() instanceof FileMacroToken);
    assertTrue(tokenizer.nextToken() instanceof DirMacroToken);
    assertTrue(tokenizer.nextToken() instanceof MethodMacroToken);
    assertTrue(tokenizer.nextToken() instanceof FunctionMacroToken);
    assertTrue(tokenizer.nextToken() instanceof ClassMacroToken);
    assertTrue(tokenizer.nextToken() instanceof NamespaceMacroToken);
    assertTrue(tokenizer.nextToken() instanceof TraitMacroToken);
}