Examples with Tokenizer org.develnext.jphp.core.tokenizer.Tokenizer used on opensource projects

Example 6 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testComments.

@Test
public void testComments() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("/** FOO BAR \n\r100500 */"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.DOCTYPE, ((CommentToken) token).getKind());
    assertEquals("FOO BAR\n\n100500", ((CommentToken) token).getComment());
    // simple
    tokenizer = new Tokenizer(new Context("// foobar \n $x"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
    assertEquals(" foobar ", ((CommentToken) token).getComment());
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertNull(tokenizer.nextToken());
    tokenizer = new Tokenizer(new Context("# // foobar \n $x"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
    assertEquals(" // foobar ", ((CommentToken) token).getComment());
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertNull(tokenizer.nextToken());
    // block
    tokenizer = new Tokenizer(new Context("/* foobar \n */"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof CommentToken);
    assertEquals(CommentToken.Kind.BLOCK, ((CommentToken) token).getKind());
}

Example 7 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testComplexOperators.

@Test
public void testComplexOperators() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("== >= <= === !== != && ||"));
    assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof GreaterOrEqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof SmallerOrEqualToken);
    assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
    assertTrue(tokenizer.nextToken() instanceof NotIdenticalExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanNotEqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanAndExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanOrExprToken);
}

Example 8 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testParseError.

@Test
public void testParseError() throws IOException {
    Throwable ex = null;
    Tokenizer tokenizer = new Tokenizer(new Context("  'foobar \n "));
    try {
        tokenizer.nextToken();
    } catch (Throwable e) {
        ex = e;
    }
    assertTrue(ex instanceof ParseException);
    TraceInfo traceInfo = ((ParseException) ex).getTraceInfo();
    assertNotNull(traceInfo);
    assertNull(traceInfo.getContext().getFileName());
    assertEquals(1, traceInfo.getStartLine());
    assertEquals(1, traceInfo.getEndLine());
    assertEquals(1, traceInfo.getStartPosition());
    assertEquals(1, traceInfo.getEndPosition());
}

Example 9 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testSplitNot.

@Test
public void testSplitNot() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("!true"));
    assertTrue(tokenizer.nextToken() instanceof BooleanNotExprToken);
    assertTrue(tokenizer.nextToken() instanceof BooleanExprToken);
}

Example 10 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testComplex.

@Test
public void testComplex() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("0==10==='30';"));
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("F =; 20;"));
    Token token;
    assertTrue((token = tokenizer.nextToken()) instanceof NameToken);
    assertEquals("F", ((NameToken) token).getName());
    assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("123foobar MAX_64Bit"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    assertEquals("foobar", token.getWord());
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertNull(tokenizer.nextToken());
    assertNull(tokenizer.nextToken());
}