Examples with TokenSource org.antlr.runtime.TokenSource used on opensource projects

Example 6 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class BaseContentAssistParser method getFollowElements.

public Collection<FE> getFollowElements(String input, boolean strict) {
    TokenSource tokenSource = createTokenSource(input);
    InternalParser parser = createParser();
    parser.setStrict(strict);
    ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
    tokens.setInitialHiddenTokens(getInitialHiddenTokens());
    parser.setTokenStream(tokens);
    IUnorderedGroupHelper helper = createUnorderedGroupHelper();
    parser.setUnorderedGroupHelper(helper);
    helper.initializeWith(parser);
    tokens.setListener(parser);
    try {
        return Lists.newArrayList(getFollowElements(parser));
    } catch (InfiniteRecursion infinite) {
        return Lists.newArrayList(parser.getFollowElements());
    }
}

Example 7 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class AbstractLexerBasedConverter method assertTokens.

/**
 * @since 2.7
 */
protected void assertTokens(T value, String result) {
    TokenSource tokenSource = getTokenSource(result);
    assertTokens(value, tokenSource, result);
}

Example 8 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-eclipse by eclipse.

the class BacktrackingLexerDocumentTokenSource method getRepairEntryData.

/**
 * @since 2.4
 */
@Override
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
    int tokenStartsAt = 0;
    int tokenInfoIdx = 0;
    TokenSource source = createTokenSource(e.fDocument.get());
    CommonToken token = (CommonToken) source.nextToken();
    // find start idx
    while (true) {
        if (token == Token.EOF_TOKEN) {
            break;
        }
        if (tokenInfoIdx >= getInternalModifyableTokenInfos().size())
            break;
        TokenInfo tokenInfo = getInternalModifyableTokenInfos().get(tokenInfoIdx);
        if (tokenInfo.getAntlrTokenType() != token.getType() || token.getStopIndex() - token.getStartIndex() + 1 != tokenInfo.getLength())
            break;
        if (tokenStartsAt + tokenInfo.getLength() > e.fOffset)
            break;
        tokenStartsAt += tokenInfo.getLength();
        tokenInfoIdx++;
        token = (CommonToken) source.nextToken();
    }
    return new RepairEntryData(tokenStartsAt, tokenInfoIdx, token, source);
}

Example 9 with TokenSource

use of org.antlr.runtime.TokenSource in project n4js by eclipse.

the class HighlightingParser method doParse.

private List<Token> doParse(CharStream in) {
    TokenSource tokenSource = createLexer(in);
    LazyTokenStream tokenStream = createTokenStream(tokenSource);
    setInitialHiddenTokens(tokenStream);
    InternalN4JSParser parser = createParser(tokenStream);
    IUnorderedGroupHelper helper = unorderedGroupHelper.get();
    if (!(helper instanceof IUnorderedGroupHelper.Null)) {
        throw new IllegalStateException("Unexpected usage of unordered groups.");
    }
    Stopwatch stopwatch = null;
    boolean debug = LOGGER.isDebugEnabled();
    // boolean debug = true;
    if (debug) {
        stopwatch = Stopwatch.createStarted();
    }
    try {
        parser.entryRuleScript();
        while (tokenStream.LT(1) != Token.EOF_TOKEN) {
            tokenStream.consume();
        }
        @SuppressWarnings("unchecked") List<Token> result = tokenStream.getTokens();
        return result;
    } catch (Exception re) {
        throw new ParseException(re.getMessage(), re);
    } finally {
        if (debug) {
            assert stopwatch != null;
            long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
            if (elapsed > 5) {
                LOGGER.warn("Coloring parser took: " + elapsed);
            }
        }
    }
}

Example 10 with TokenSource

use of org.antlr.runtime.TokenSource in project antlr4 by tunnelvisionlabs.

the class TestASTStructure method execParser.

public Object execParser(String ruleName, String input, int scriptLine) throws Exception {
    ANTLRStringStream is = new ANTLRStringStream(input);
    Class<? extends TokenSource> lexerClass = Class.forName(lexerClassName).asSubclass(TokenSource.class);
    Constructor<? extends TokenSource> lexConstructor = lexerClass.getConstructor(CharStream.class);
    TokenSource lexer = lexConstructor.newInstance(is);
    is.setLine(scriptLine);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Class<? extends Parser> parserClass = Class.forName(parserClassName).asSubclass(Parser.class);
    Constructor<? extends Parser> parConstructor = parserClass.getConstructor(TokenStream.class);
    Parser parser = parConstructor.newInstance(tokens);
    // set up customized tree adaptor if necessary
    if (adaptorClassName != null) {
        Method m = parserClass.getMethod("setTreeAdaptor", TreeAdaptor.class);
        Class<? extends TreeAdaptor> adaptorClass = Class.forName(adaptorClassName).asSubclass(TreeAdaptor.class);
        m.invoke(parser, adaptorClass.newInstance());
    }
    Method ruleMethod = parserClass.getMethod(ruleName);
    // INVOKE RULE
    return ruleMethod.invoke(parser);
}