Examples with Tokenizer org.develnext.jphp.core.tokenizer.Tokenizer used on opensource projects

Example 16 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class JvmCompilerCase method getSyntax.

protected SyntaxAnalyzer getSyntax(Context context) {
    Tokenizer tokenizer = null;
    try {
        tokenizer = new Tokenizer(context);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    environment.scope.setLangMode(LangMode.DEFAULT);
    return new SyntaxAnalyzer(environment, tokenizer);
}

Example 17 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class SimpleExprTest method testSimpleCall.

@Test
public void testSimpleCall() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("myCall(1 * 2, func(3, 2), 4);"));
    SyntaxAnalyzer analyzer = new SyntaxAnalyzer(environment, tokenizer);
    List<Token> tokens = analyzer.getTree();
    Assert.assertTrue(tokens.size() == 1);
    Assert.assertTrue(tokens.get(0) instanceof ExprStmtToken);
    ExprStmtToken expr = (ExprStmtToken) tokens.get(0);
    tokens = expr.getTokens();
    Assert.assertTrue(tokens.size() == 1);
    Assert.assertTrue(tokens.get(0) instanceof CallExprToken);
    CallExprToken call = (CallExprToken) expr.getTokens().get(0);
    Assert.assertTrue(call.getName() instanceof NameToken);
    Assert.assertEquals("myCall", ((NameToken) call.getName()).getName());
    Assert.assertTrue(call.getParameters().size() == 3);
    Assert.assertTrue(call.getParameters().get(0).getTokens().size() == 3);
    Assert.assertTrue(call.getParameters().get(2).getTokens().size() == 1);
    ExprStmtToken param = call.getParameters().get(1);
    Assert.assertTrue(param.getTokens().size() == 1);
    Assert.assertTrue(param.getTokens().get(0) instanceof CallExprToken);
    CallExprToken subCall = (CallExprToken) param.getTokens().get(0);
    Assert.assertTrue(subCall.getParameters().size() == 2);
}

Example 18 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class SyntaxAnalyzerTest method testSimple.

@Test
public void testSimple() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("foobar;"));
    SyntaxAnalyzer analyzer = new SyntaxAnalyzer(environment, tokenizer);
    Assert.assertTrue(analyzer.getTree().size() == 1);
    Assert.assertTrue(analyzer.getTree().listIterator().next() instanceof ExprStmtToken);
}

Example 19 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class CLI method checkSyntax.

protected void checkSyntax(String filename) throws Throwable {
    Launcher launcher = new Launcher("jphp.conf", args);
    launcher.run(false, true);
    File file = new File(filename);
    Environment environment = new Environment(launcher.getCompileScope(), output);
    Context context = new Context(file);
    try {
        SyntaxAnalyzer analyzer = new SyntaxAnalyzer(environment, new Tokenizer(context));
        analyzer.getTree();
        output.println(String.format("No syntax errors detected in %s", filename));
    } catch (Exception e) {
        environment.catchUncaught(e);
    } catch (Throwable throwable) {
        throw new RuntimeException(throwable);
    } finally {
        try {
            environment.doFinal();
        } catch (Throwable throwable) {
            throw new RuntimeException(throwable);
        }
    }
}

Example 20 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class SimpleExprGenerator method processString.

protected Token processString(StringExprToken string) {
    if (string.getSegments().isEmpty()) {
        if (string.getQuote() == StringExprToken.Quote.SHELL) {
            return new ShellExecExprToken(string.getMeta(), Arrays.<Token>asList(string));
        }
        return string;
    }
    List<Token> tokens = new ArrayList<Token>();
    int i = 0;
    String value = string.getValue();
    TokenMeta meta = string.getMeta();
    for (StringExprToken.Segment segment : string.getSegments()) {
        String prev = value.substring(i, segment.from);
        if (!prev.isEmpty()) {
            StringExprToken item = new StringExprToken(new TokenMeta(prev, meta.getStartLine() + i, meta.getEndLine(), meta.getStartLine(), meta.getEndLine()), StringExprToken.Quote.SINGLE);
            tokens.add(item);
        }
        String dynamic = value.substring(segment.from, segment.to);
        if (!segment.isVariable)
            dynamic = dynamic.substring(1, dynamic.length() - 1);
        Tokenizer tokenizer = new Tokenizer(dynamic + ";", analyzer.getContext());
        try {
            SyntaxAnalyzer syntaxAnalyzer = new SyntaxAnalyzer(analyzer.getEnvironment(), tokenizer, analyzer.getFunction());
            List<Token> tree = syntaxAnalyzer.getTree();
            analyzer.getScope().addVariables(syntaxAnalyzer.getScope().getVariables());
            assert tree.size() > 0;
            Token item = tree.get(0);
            if (!(item instanceof ExprStmtToken))
                unexpectedToken(item);
            ExprStmtToken expr = (ExprStmtToken) item;
            if (expr.isSingle()) {
                tokens.add(expr.getSingle());
            } else
                tokens.add(expr);
        } catch (ParseException e) {
            TraceInfo oldTrace = e.getTraceInfo();
            e.setTraceInfo(new TraceInfo(analyzer.getContext(), meta.getStartLine() + oldTrace.getStartLine(), meta.getEndLine() + oldTrace.getEndLine(), meta.getStartLine() + oldTrace.getStartLine(), meta.getEndLine() + oldTrace.getEndLine()));
            throw e;
        }
        i = segment.to;
    }
    String prev = value.substring(i);
    if (!prev.isEmpty()) {
        StringExprToken item = new StringExprToken(new TokenMeta(prev, meta.getStartLine() + i, meta.getEndLine(), meta.getStartLine(), meta.getEndLine()), StringExprToken.Quote.SINGLE);
        tokens.add(item);
    }
    if (string.getQuote() == StringExprToken.Quote.SHELL) {
        return new ShellExecExprToken(meta, tokens);
    }
    StringBuilderExprToken result = new StringBuilderExprToken(meta, tokens);
    result.setBinary(string.isBinary());
    return result;
}