Search in sources :

Example 11 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testBraces.

@Test
public void testBraces() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context(" :: ->foobar('a', 1, 3.0);"));
    assertTrue(tokenizer.nextToken() instanceof StaticAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof DynamicAccessExprToken);
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof CommaToken);
    assertTrue(tokenizer.nextToken() instanceof DoubleExprToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
}
Also used : Context(php.runtime.env.Context) SemicolonToken(org.develnext.jphp.core.tokenizer.token.SemicolonToken) Tokenizer(org.develnext.jphp.core.tokenizer.Tokenizer) Test(org.junit.Test)

Example 12 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testBug122.

@Test
public void testBug122() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("<<<ESC\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "<?php\n" + "ESC;\n"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals("\n\n\n\n\n<?php", ((StringExprToken) token).getValue());
}
Also used : Context(php.runtime.env.Context) Token(org.develnext.jphp.core.tokenizer.token.Token) CommentToken(org.develnext.jphp.core.tokenizer.token.CommentToken) SemicolonToken(org.develnext.jphp.core.tokenizer.token.SemicolonToken) Tokenizer(org.develnext.jphp.core.tokenizer.Tokenizer) Test(org.junit.Test)

Example 13 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testSimpleOperators.

@Test
public void testSimpleOperators() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("= + - / * % . and or new && || ! xor **"));
    assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
    assertTrue(tokenizer.nextToken() instanceof PlusExprToken);
    assertTrue(tokenizer.nextToken() instanceof MinusExprToken);
    assertTrue(tokenizer.nextToken() instanceof DivExprToken);
    assertTrue(tokenizer.nextToken() instanceof MulExprToken);
    assertTrue(tokenizer.nextToken() instanceof ModExprToken);
    assertTrue(tokenizer.nextToken() instanceof ConcatExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanAnd2ExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanOr2ExprToken);
    assertTrue(tokenizer.nextToken() instanceof NewExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanAndExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanOrExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanNotExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanXorExprToken);
    assertTrue(tokenizer.nextToken() instanceof PowExprToken);
}
Also used : Context(php.runtime.env.Context) Tokenizer(org.develnext.jphp.core.tokenizer.Tokenizer) Test(org.junit.Test)

Example 14 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testHeredoc.

@Test
public void testHeredoc() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("<<<DOC\n <foobar> \nDOC;\n"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" <foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<\"DOC\"\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<'DOC'\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \\n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("a(<<<DOC\n<foobar>\nDOC\n)\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof BraceExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals("<foobar>", ((StringExprToken) token).getValue());
}
Also used : Context(php.runtime.env.Context) Token(org.develnext.jphp.core.tokenizer.token.Token) CommentToken(org.develnext.jphp.core.tokenizer.token.CommentToken) SemicolonToken(org.develnext.jphp.core.tokenizer.token.SemicolonToken) Tokenizer(org.develnext.jphp.core.tokenizer.Tokenizer) Test(org.junit.Test)

Example 15 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testMacro.

@Test
public void testMacro() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("__LINE__ __FILE__ __DIR__ __METHOD__ __FUNCTION__ __CLASS__ __NAMESPACE__ __TRAIT__"));
    assertTrue(tokenizer.nextToken() instanceof LineMacroToken);
    assertTrue(tokenizer.nextToken() instanceof FileMacroToken);
    assertTrue(tokenizer.nextToken() instanceof DirMacroToken);
    assertTrue(tokenizer.nextToken() instanceof MethodMacroToken);
    assertTrue(tokenizer.nextToken() instanceof FunctionMacroToken);
    assertTrue(tokenizer.nextToken() instanceof ClassMacroToken);
    assertTrue(tokenizer.nextToken() instanceof NamespaceMacroToken);
    assertTrue(tokenizer.nextToken() instanceof TraitMacroToken);
}
Also used : Context(php.runtime.env.Context) Tokenizer(org.develnext.jphp.core.tokenizer.Tokenizer) Test(org.junit.Test)

Aggregations

Tokenizer (org.develnext.jphp.core.tokenizer.Tokenizer)30 Context (php.runtime.env.Context)27 Test (org.junit.Test)23 Token (org.develnext.jphp.core.tokenizer.token.Token)11 SemicolonToken (org.develnext.jphp.core.tokenizer.token.SemicolonToken)9 CommentToken (org.develnext.jphp.core.tokenizer.token.CommentToken)8 SyntaxAnalyzer (org.develnext.jphp.core.syntax.SyntaxAnalyzer)6 IOException (java.io.IOException)4 ExprStmtToken (org.develnext.jphp.core.tokenizer.token.stmt.ExprStmtToken)4 FunctionStmtToken (org.develnext.jphp.core.tokenizer.token.stmt.FunctionStmtToken)3 ParseException (php.runtime.exceptions.ParseException)3 TraceInfo (php.runtime.env.TraceInfo)2 File (java.io.File)1 BigInteger (java.math.BigInteger)1 JvmCompiler (org.develnext.jphp.core.compiler.jvm.JvmCompiler)1 TokenMeta (org.develnext.jphp.core.tokenizer.TokenMeta)1 BreakToken (org.develnext.jphp.core.tokenizer.token.BreakToken)1 ColonToken (org.develnext.jphp.core.tokenizer.token.ColonToken)1 MinusExprToken (org.develnext.jphp.core.tokenizer.token.expr.operator.MinusExprToken)1 ValueIfElseToken (org.develnext.jphp.core.tokenizer.token.expr.operator.ValueIfElseToken)1