Examples with Token org.develnext.jphp.core.tokenizer.token.Token used on opensource projects

Example 26 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenizerTest method testComplex.

@Test
public void testComplex() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("0==10==='30';"));
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("F =; 20;"));
    Token token;
    assertTrue((token = tokenizer.nextToken()) instanceof NameToken);
    assertEquals("F", ((NameToken) token).getName());
    assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
    assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
    tokenizer = new Tokenizer(new Context("123foobar MAX_64Bit"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    assertEquals("foobar", token.getWord());
    assertTrue(tokenizer.nextToken() instanceof NameToken);
    assertNull(tokenizer.nextToken());
    assertNull(tokenizer.nextToken());
}

Example 27 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenizerTest method testHeredoc.

@Test
public void testHeredoc() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("<<<DOC\n <foobar> \nDOC;\n"));
    Token token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" <foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<\"DOC\"\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("<<<'DOC'\n \\n<foobar> \nDOC;\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals(" \\n<foobar> ", ((StringExprToken) token).getValue());
    tokenizer = new Tokenizer(new Context("a(<<<DOC\n<foobar>\nDOC\n)\n"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof NameToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof BraceExprToken);
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
    assertEquals("<foobar>", ((StringExprToken) token).getValue());
}

Example 28 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class TokenizerAssert method nextIsStringToken.

public TokenizerAssert nextIsStringToken(String expectedValue, StringExprToken.Quote quote) {
    Token actual = this.actual.nextToken();
    isStringTokenWithValue(expectedValue, actual);
    assertThat(((StringExprToken) actual).getQuote()).isEqualTo(quote);
    return this;
}

Example 29 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SyntaxAnalyzer method process.

protected void process() {
    tokenizer.reset();
    tree.clear();
    Token token;
    try {
        while ((token = tokenizer.nextToken()) != null) {
            if (token instanceof CommentToken) {
                if (((CommentToken) token).getKind() != CommentToken.Kind.DOCTYPE)
                    continue;
            }
            tokens.add(token);
        }
    } catch (ParseException e) {
        this.getEnvironment().error(e.getTraceInfo(), ErrorType.E_PARSE, e.getMessage());
    }
    /*if (tokenizer.hasDirective("mode")){
            Directive mode = tokenizer.getDirective("mode");
            try {
                this.langMode = LangMode.valueOf(mode.value.toUpperCase());
            } catch (IllegalArgumentException e){
                environment.warning(
                        mode.trace, "Invalid value '%s' for directive 'mode'", mode.value
                );
            }
        }*/
    ListIterator<Token> iterator = tokens.listIterator();
    tree = process(iterator);
}

Example 30 with Token

use of org.develnext.jphp.core.tokenizer.token.Token in project jphp by jphp-compiler.

the class SyntaxAnalyzer method process.

public List<Token> process(ListIterator<Token> iterator) {
    List<Token> result = new ArrayList<Token>();
    while (iterator.hasNext()) {
        Token gen = processNext(iterator);
        if (gen instanceof NamespaceStmtToken) {
            List<Token> tree = ((NamespaceStmtToken) gen).getTree();
            ((NamespaceStmtToken) gen).setTree(null);
            result.add(gen);
            registerToken(gen);
            result.addAll(tree);
            if (!((NamespaceStmtToken) gen).isTokenRegistered()) {
                for (Token el : tree) {
                    registerToken(el);
                }
            }
        } else {
            result.add(gen);
            registerToken(gen);
        }
    }
    return result;
}