Examples with TokenSource org.antlr.runtime.TokenSource used on opensource projects

Example 1 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-eclipse by eclipse.

the class BacktrackingLexerDocumentTokenSource method getRepairEntryData.

/**
 * @since 2.4
 */
@Override
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
    int tokenStartsAt = 0;
    int tokenInfoIdx = 0;
    TokenSource source = createTokenSource(e.fDocument.get());
    CommonToken token = (CommonToken) source.nextToken();
    // find start idx
    while (true) {
        if (token == Token.EOF_TOKEN) {
            break;
        }
        if (tokenInfoIdx >= getInternalModifyableTokenInfos().size())
            break;
        TokenInfo tokenInfo = getInternalModifyableTokenInfos().get(tokenInfoIdx);
        if (tokenInfo.getAntlrTokenType() != token.getType() || token.getStopIndex() - token.getStartIndex() + 1 != tokenInfo.getLength())
            break;
        if (tokenStartsAt + tokenInfo.getLength() > e.fOffset)
            break;
        tokenStartsAt += tokenInfo.getLength();
        tokenInfoIdx++;
        token = (CommonToken) source.nextToken();
    }
    return new RepairEntryData(tokenStartsAt, tokenInfoIdx, token, source);
}

Example 2 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class BaseContentAssistParser method getFollowElements.

public Collection<FE> getFollowElements(String input, boolean strict) {
    TokenSource tokenSource = createTokenSource(input);
    InternalParser parser = createParser();
    parser.setStrict(strict);
    ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
    tokens.setInitialHiddenTokens(getInitialHiddenTokens());
    parser.setTokenStream(tokens);
    IUnorderedGroupHelper helper = createUnorderedGroupHelper();
    parser.setUnorderedGroupHelper(helper);
    helper.initializeWith(parser);
    tokens.setListener(parser);
    try {
        return Lists.newArrayList(getFollowElements(parser));
    } catch (InfiniteRecursion infinite) {
        return Lists.newArrayList(parser.getFollowElements());
    }
}

Example 3 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class BaseContentAssistParser method getFollowElements.

public Collection<FE> getFollowElements(IParseResult parseResult, EntryPointFinder entryPointFinder, int offset, boolean strict) {
    ICompositeNode entryPoint = entryPointFinder.findEntryPoint(parseResult, offset);
    if (entryPoint != null) {
        String parseMe = getTextToParse(parseResult, entryPoint, offset);
        TokenSource tokenSource = createTokenSource(parseMe);
        InternalParser parser = createParser();
        parser.setStrict(strict);
        ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
        tokens.setInitialHiddenTokens(getInitialHiddenTokens());
        parser.setTokenStream(tokens);
        IUnorderedGroupHelper helper = createUnorderedGroupHelper();
        parser.setUnorderedGroupHelper(helper);
        helper.initializeWith(parser);
        tokens.setListener(parser);
        try {
            Collection<FE> followElements = getFollowElements(parser, getEntryGrammarElement(entryPoint));
            return Lists.newArrayList(followElements);
        } catch (InfiniteRecursion infinite) {
            return Lists.newArrayList(parser.getFollowElements());
        }
    } else {
        String text = parseResult.getRootNode().getText();
        String parseMe = text.substring(0, offset);
        initializeFor(NodeModelUtils.getEntryParserRule(parseResult.getRootNode()));
        Collection<FE> followElements = getFollowElements(parseMe, strict);
        return followElements;
    }
}

Example 4 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class AbstractAntlrParser method doParse.

protected IParseResult doParse(String ruleName, CharStream in, NodeModelBuilder nodeModelBuilder, int initialLookAhead) {
    TokenSource tokenSource = createLexer(in);
    XtextTokenStream tokenStream = createTokenStream(tokenSource);
    tokenStream.initCurrentLookAhead(initialLookAhead);
    setInitialHiddenTokens(tokenStream);
    AbstractInternalAntlrParser parser = createParser(tokenStream);
    parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
    parser.setSyntaxErrorProvider(getSyntaxErrorProvider());
    parser.setNodeModelBuilder(nodeModelBuilder);
    parser.setSemanticModelBuilder(getElementFactory());
    IUnorderedGroupHelper helper = getUnorderedGroupHelper().get();
    parser.setUnorderedGroupHelper(helper);
    helper.initializeWith(parser);
    try {
        if (ruleName != null)
            return parser.parse(ruleName);
        return parser.parse();
    } catch (Exception re) {
        throw new ParseException(re.getMessage(), re);
    }
}

Example 5 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-core by eclipse.

the class AbstractLexerBasedConverter method assertTokens.

/**
 * @since 2.7
 */
protected void assertTokens(T value, String result) {
    TokenSource tokenSource = getTokenSource(result);
    assertTokens(value, tokenSource, result);
}