Examples with Tokenizer org.develnext.jphp.core.tokenizer.Tokenizer used on opensource projects

Example 1 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class EvalFunctions method eval.

public static Memory eval(Environment env, TraceInfo trace, @Runtime.GetLocals ArrayMemory locals, String code) throws Throwable {
    Context context = new Context(code);
    try {
        Tokenizer tokenizer = new Tokenizer(context);
        SyntaxAnalyzer analyzer = syntaxAnalyzer.get();
        analyzer.reset(env, tokenizer);
        AbstractCompiler compiler = new JvmCompiler(env, context, analyzer);
        ModuleEntity module = compiler.compile();
        env.scope.loadModule(module);
        env.registerModule(module);
        return module.include(env, locals);
    } catch (ErrorException e) {
        if (e.getType() == ErrorType.E_PARSE) {
            if (env.isHandleErrors(ErrorType.E_PARSE))
                throw new ParseException(evalErrorMessage(e), trace);
        } else
            env.error(trace, e.getType(), evalErrorMessage(e));
    }
    return Memory.FALSE;
}

Example 2 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testStmt.

@Test
public void testStmt() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("class function private public protected static final try catch for if foreach switch while " + "default return declare case do else elseif endif endfor endforeach endwhile endswitch " + "abstract use namespace finally extends implements global"));
    assertTrue(tokenizer.nextToken() instanceof ClassStmtToken);
    assertTrue(tokenizer.nextToken() instanceof FunctionStmtToken);
    assertTrue(tokenizer.nextToken() instanceof PrivateStmtToken);
    assertTrue(tokenizer.nextToken() instanceof PublicStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ProtectedStmtToken);
    assertTrue(tokenizer.nextToken() instanceof StaticExprToken);
    assertTrue(tokenizer.nextToken() instanceof FinalStmtToken);
    assertTrue(tokenizer.nextToken() instanceof TryStmtToken);
    assertTrue(tokenizer.nextToken() instanceof CatchStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ForStmtToken);
    assertTrue(tokenizer.nextToken() instanceof IfStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ForeachStmtToken);
    assertTrue(tokenizer.nextToken() instanceof SwitchStmtToken);
    assertTrue(tokenizer.nextToken() instanceof WhileStmtToken);
    assertTrue(tokenizer.nextToken() instanceof DefaultStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ReturnStmtToken);
    assertTrue(tokenizer.nextToken() instanceof DeclareStmtToken);
    assertTrue(tokenizer.nextToken() instanceof CaseStmtToken);
    assertTrue(tokenizer.nextToken() instanceof DoStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ElseStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ElseIfStmtToken);
    assertTrue(tokenizer.nextToken() instanceof EndifStmtToken);
    assertTrue(tokenizer.nextToken() instanceof EndforStmtToken);
    assertTrue(tokenizer.nextToken() instanceof EndforeachStmtToken);
    assertTrue(tokenizer.nextToken() instanceof EndwhileStmtToken);
    assertTrue(tokenizer.nextToken() instanceof EndswitchStmtToken);
    assertTrue(tokenizer.nextToken() instanceof AbstractStmtToken);
    assertTrue(tokenizer.nextToken() instanceof NamespaceUseStmtToken);
    assertTrue(tokenizer.nextToken() instanceof NamespaceStmtToken);
    assertTrue(tokenizer.nextToken() instanceof FinallyStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ExtendsStmtToken);
    assertTrue(tokenizer.nextToken() instanceof ImplementsStmtToken);
    assertTrue(tokenizer.nextToken() instanceof GlobalStmtToken);
}

Example 3 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testVarVar.

@Test
public void testVarVar() throws IOException {
    Tokenizer tokenizer = new Tokenizer(new Context("$$foo $ $bar $$$foobar"));
    assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
    assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
    assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
    tokenizer = new Tokenizer(new Context("${'foo;6bar'}"));
    assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
    assertTrue(tokenizer.nextToken() instanceof StringExprToken);
    assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
}

Example 4 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class ASMExpressionTest method getASMExpression.

private ASMExpression getASMExpression(String expr) {
    environment.scope.setLangMode(LangMode.DEFAULT);
    Tokenizer tokenizer = null;
    try {
        tokenizer = new Tokenizer(context = new Context(expr + ";"));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    SyntaxAnalyzer analyzer = new SyntaxAnalyzer(environment, tokenizer);
    Assert.assertTrue(analyzer.getTree().size() == 1);
    Assert.assertTrue(analyzer.getTree().get(0) instanceof ExprStmtToken);
    return new ASMExpression(null, context, (ExprStmtToken) analyzer.getTree().get(0));
}

Example 5 with Tokenizer

use of org.develnext.jphp.core.tokenizer.Tokenizer in project jphp by jphp-compiler.

the class TokenizerTest method testScalarTokens.

@Test
public void testScalarTokens() throws IOException {
    Token token;
    Tokenizer tokenizer = new Tokenizer(new Context("10 3.3 'foo' '' \"xyz\" 0xCC 0b0011 true false"));
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(10L, ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof DoubleExprToken);
    assertEquals(3.3, ((DoubleExprToken) token).getValue(), 0.01);
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
    assertEquals("foo", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
    assertEquals("", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof StringExprToken);
    assertEquals(StringExprToken.Quote.DOUBLE, ((StringExprToken) token).getQuote());
    assertEquals("xyz", ((StringExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(new BigInteger("CC", 16).longValue(), ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof IntegerExprToken);
    assertEquals(new BigInteger("0011", 2).longValue(), ((IntegerExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof BooleanExprToken);
    assertEquals(true, ((BooleanExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertTrue(token instanceof BooleanExprToken);
    assertEquals(false, ((BooleanExprToken) token).getValue());
    token = tokenizer.nextToken();
    assertNull(token);
}