Examples with LazyTokenStream org.eclipse.n4js.parser.LazyTokenStream used on opensource projects

Example 1 with LazyTokenStream

use of org.eclipse.n4js.parser.LazyTokenStream in project n4js by eclipse.

the class HighlightingParser method doParse.

private List<Token> doParse(CharStream in) {
    TokenSource tokenSource = createLexer(in);
    LazyTokenStream tokenStream = createTokenStream(tokenSource);
    setInitialHiddenTokens(tokenStream);
    InternalN4JSParser parser = createParser(tokenStream);
    IUnorderedGroupHelper helper = unorderedGroupHelper.get();
    if (!(helper instanceof IUnorderedGroupHelper.Null)) {
        throw new IllegalStateException("Unexpected usage of unordered groups.");
    }
    Stopwatch stopwatch = null;
    boolean debug = LOGGER.isDebugEnabled();
    // boolean debug = true;
    if (debug) {
        stopwatch = Stopwatch.createStarted();
    }
    try {
        parser.entryRuleScript();
        while (tokenStream.LT(1) != Token.EOF_TOKEN) {
            tokenStream.consume();
        }
        @SuppressWarnings("unchecked") List<Token> result = tokenStream.getTokens();
        return result;
    } catch (Exception re) {
        throw new ParseException(re.getMessage(), re);
    } finally {
        if (debug) {
            assert stopwatch != null;
            long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
            if (elapsed > 5) {
                LOGGER.warn("Coloring parser took: " + elapsed);
            }
        }
    }
}

Example 2 with LazyTokenStream

use of org.eclipse.n4js.parser.LazyTokenStream in project n4js by eclipse.

the class InternalHighlightingParser method announce.

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
    if (start != null && start != Token.EOF_TOKEN) {
        if (start == stop) {
            announce(start, element);
        } else {
            CommonToken castedStart = (CommonToken) start;
            if (stop == null) {
                // possible error condition
                if (start.getTokenIndex() == state.lastErrorIndex) {
                    return;
                }
            }
            CommonToken castedEnd = (CommonToken) stop;
            Integer newType = rewriter.rewrite(castedStart, element);
            if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
                LazyTokenStream castedInput = (LazyTokenStream) this.input;
                for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
                    Token token = castedInput.get(i);
                    if (token.getChannel() != Token.HIDDEN_CHANNEL)
                        token.setType(newType);
                }
                castedEnd.setType(newType);
            }
        }
    }
}