Examples with CoffeeSymbol com.aptana.editor.coffee.parsing.lexer.CoffeeSymbol used on opensource projects

Example 1 with CoffeeSymbol

use of com.aptana.editor.coffee.parsing.lexer.CoffeeSymbol in project coffeescript-eclipse by adamschmideg.

the class Helper method checkTokenStarts.

/**
	 * Check if the start location of successive tokens are increasing
	 * @param tokens
	 * @return the index where the assumption fails, or -1 when start locations are increasing
	 */
public static int checkTokenStarts(List<CoffeeSymbol> tokens) {
    int problemIndex = -1;
    for (int i = 1; i < tokens.size(); i++) {
        CoffeeSymbol prev = tokens.get(i - 1);
        CoffeeSymbol current = tokens.get(i);
        //System.out.println(current.debugString());
        if (prev.getStart() > current.getStart()) {
            problemIndex = i;
            System.out.println("\t\t!!! at " + prev + ", " + current);
        //throw new RuntimeException("bad " + tokens);				
        }
    }
    return problemIndex;
//System.out.println("---\n");
}

Example 2 with CoffeeSymbol

use of com.aptana.editor.coffee.parsing.lexer.CoffeeSymbol in project coffeescript-eclipse by adamschmideg.

the class Lexer method nextToken.

/**
	 * Get next token.  If an exception is thrown by the underlying lexer,
	 * keep calling it, and append an invalid token at the very end.
	 */
@Override
public Token nextToken() {
    Token token = null;
    CoffeeSymbol symbol = null;
    try {
        symbol = aptanaScanner.nextToken();
        if (symbol == null || symbol.getId() < 0) {
            logger.warn("Unexpected symbol " + symbol, new Exception());
            token = CommonToken.INVALID_TOKEN;
        } else if (symbol.getId() == Terminals.EOF) {
            token = CommonToken.EOF_TOKEN;
        } else {
            token = new BeaverToken(symbol);
            if (((CommonToken) token).getStopIndex() >= input.size()) {
                assert false : "Token stop index overflows " + symbol + " in:\n<<<" + content + ">>>";
            }
        }
    } catch (Exception e) {
        // Xtext wants token to be CommonToken, INVALID_TOKEN_TYPE, and HIDDEN_CHANNEL
        String text = e.getLocalizedMessage();
        if (text == null)
            text = "simply " + e.getClass().getSimpleName();
        CommonToken ct = new CommonToken(Token.INVALID_TOKEN_TYPE, text);
        ct.setChannel(Token.HIDDEN_CHANNEL);
        if (prevToken != null) {
            int start = prevToken.getStopIndex() + 1;
            // TODO: get more informative errors with length of token
            int stop = start + 1;
            ct.setStartIndex(start);
            ct.setStopIndex(stop);
        }
        token = ct;
    }
    token.setTokenIndex(tokenIndex);
    if (symbol != null && symbol.hidden)
        token.setChannel(Token.HIDDEN_CHANNEL);
    tokenIndex++;
    if (token instanceof CommonToken) {
        if (prevToken != null && token.getType() > 0) {
            if (((CommonToken) token).getStartIndex() < prevToken.getStartIndex()) {
                assert false : "Position not follows, prevToken: " + prevToken + ", token: " + token;
            }
        }
        prevToken = (CommonToken) token;
    }
    logger.debug("token: " + token);
    return token;
}