Examples with TokenSource org.antlr.runtime.TokenSource used on opensource projects

Example 11 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-eclipse by eclipse.

the class BacktrackingLexerDocumentTokenSource method getRepairEntryData.

/**
 * @since 2.4
 */
@Override
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
    int tokenStartsAt = 0;
    int tokenInfoIdx = 0;
    TokenSource source = createTokenSource(e.fDocument.get());
    CommonToken token = (CommonToken) source.nextToken();
    // find start idx
    while (true) {
        if (token == Token.EOF_TOKEN) {
            break;
        }
        if (tokenInfoIdx >= getInternalModifyableTokenInfos().size())
            break;
        TokenInfo tokenInfo = getInternalModifyableTokenInfos().get(tokenInfoIdx);
        if (tokenInfo.getAntlrTokenType() != token.getType() || token.getStopIndex() - token.getStartIndex() + 1 != tokenInfo.getLength())
            break;
        if (tokenStartsAt + tokenInfo.getLength() > e.fOffset)
            break;
        tokenStartsAt += tokenInfo.getLength();
        tokenInfoIdx++;
        token = (CommonToken) source.nextToken();
    }
    return new RepairEntryData(tokenStartsAt, tokenInfoIdx, token, source);
}

Example 12 with TokenSource

use of org.antlr.runtime.TokenSource in project InspectorClient by zhkl0228.

the class SmaliTestUtils method compileSmali.

public static ClassDef compileSmali(String smaliText, int apiLevel) throws RecognitionException, IOException {
    CommonTokenStream tokens;
    LexerErrorInterface lexer;
    DexBuilder dexBuilder = new DexBuilder(Opcodes.forApi(apiLevel));
    Reader reader = new StringReader(smaliText);
    lexer = new smaliFlexLexer(reader);
    tokens = new CommonTokenStream((TokenSource) lexer);
    smaliParser parser = new smaliParser(tokens);
    parser.setVerboseErrors(true);
    parser.setAllowOdex(false);
    parser.setApiLevel(apiLevel);
    smaliParser.smali_file_return result = parser.smali_file();
    if (parser.getNumberOfSyntaxErrors() > 0 || lexer.getNumberOfSyntaxErrors() > 0) {
        throw new RuntimeException("Error occured while compiling text");
    }
    CommonTree t = result.getTree();
    CommonTreeNodeStream treeStream = new CommonTreeNodeStream(t);
    treeStream.setTokenStream(tokens);
    smaliTreeWalker dexGen = new smaliTreeWalker(treeStream);
    dexGen.setApiLevel(apiLevel);
    dexGen.setVerboseErrors(true);
    dexGen.setDexBuilder(dexBuilder);
    dexGen.smali_file();
    if (dexGen.getNumberOfSyntaxErrors() > 0) {
        throw new RuntimeException("Error occured while compiling text");
    }
    MemoryDataStore dataStore = new MemoryDataStore();
    dexBuilder.writeTo(dataStore);
    DexBackedDexFile dexFile = new DexBackedDexFile(Opcodes.forApi(apiLevel), dataStore.getData());
    return Iterables.getFirst(dexFile.getClasses(), null);
}

Example 13 with TokenSource

use of org.antlr.runtime.TokenSource in project n4js by eclipse.

the class CustomN4JSParser method getFollowElements.

@SuppressWarnings("hiding")
@Override
public Collection<FollowElement> getFollowElements(FollowElement element) {
    if (element.getLookAhead() <= 1)
        throw new IllegalArgumentException("lookahead may not be less than or equal to 1");
    Collection<FollowElement> result = new ArrayList<>();
    Collection<AbstractElement> elementsToParse = this.reflective("getElementsToParse", element);
    for (AbstractElement elementToParse : elementsToParse) {
        // fix is here
        elementToParse = unwrapSingleElementGroups(elementToParse);
        // done
        String ruleName = getRuleName(elementToParse);
        String[][] allRuleNames = reflective("getRequiredRuleNames", ruleName, element.getParamStack(), elementToParse);
        for (String[] ruleNames : allRuleNames) {
            for (int i = 0; i < ruleNames.length; i++) {
                AbstractInternalContentAssistParser parser = createParser();
                parser.setUnorderedGroupHelper(getUnorderedGroupHelper().get());
                parser.getUnorderedGroupHelper().initializeWith(parser);
                final Iterator<LookAheadTerminal> iter = element.getLookAheadTerminals().iterator();
                ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(new TokenSource() {

                    @Override
                    public Token nextToken() {
                        if (iter.hasNext()) {
                            LookAheadTerminal lookAhead = iter.next();
                            return lookAhead.getToken();
                        }
                        return Token.EOF_TOKEN;
                    }

                    @Override
                    public String getSourceName() {
                        return "LookAheadTerminalTokenSource";
                    }
                }, parser);
                parser.setTokenStream(tokens);
                tokens.setListener(parser);
                parser.getGrammarElements().addAll(element.getTrace());
                parser.getGrammarElements().add(elementToParse);
                parser.getLocalTrace().addAll(element.getLocalTrace());
                parser.getLocalTrace().add(elementToParse);
                parser.getParamStack().addAll(element.getParamStack());
                if (elementToParse instanceof UnorderedGroup && element.getGrammarElement() == elementToParse) {
                    UnorderedGroup group = (UnorderedGroup) elementToParse;
                    final IUnorderedGroupHelper helper = parser.getUnorderedGroupHelper();
                    helper.enter(group);
                    for (AbstractElement consumed : element.getHandledUnorderedGroupElements()) {
                        parser.before(consumed);
                        helper.select(group, group.getElements().indexOf(consumed));
                        helper.returnFromSelection(group);
                        parser.after(consumed);
                    }
                    parser.setUnorderedGroupHelper(new IUnorderedGroupHelper() {

                        boolean first = true;

                        @Override
                        public void initializeWith(BaseRecognizer recognizer) {
                            helper.initializeWith(recognizer);
                        }

                        @Override
                        public void enter(UnorderedGroup group) {
                            if (!first)
                                helper.enter(group);
                            first = false;
                        }

                        @Override
                        public void leave(UnorderedGroup group) {
                            helper.leave(group);
                        }

                        @Override
                        public boolean canSelect(UnorderedGroup group, int index) {
                            return helper.canSelect(group, index);
                        }

                        @Override
                        public void select(UnorderedGroup group, int index) {
                            helper.select(group, index);
                        }

                        @Override
                        public void returnFromSelection(UnorderedGroup group) {
                            helper.returnFromSelection(group);
                        }

                        @Override
                        public boolean canLeave(UnorderedGroup group) {
                            return helper.canLeave(group);
                        }

                        @Override
                        public UnorderedGroupState snapShot(UnorderedGroup... groups) {
                            return helper.snapShot(groups);
                        }
                    });
                }
                Collection<FollowElement> elements = reflective("getFollowElements", parser, elementToParse, ruleNames, i);
                result.addAll(elements);
            }
        }
    }
    return result;
}

Example 14 with TokenSource

use of org.antlr.runtime.TokenSource in project n4js by eclipse.

the class CustomN4JSParser method getFollowElements.

/**
 * @param node
 *            the root node of the model to parse
 * @param startOffset
 *            the start offset to consider
 * @param endOffset
 *            the exclusive end offset
 * @param strict
 *            if true the parser will not use error recovery on the very last token of the input.
 * @return a collection of follow elements.
 */
public Collection<FollowElement> getFollowElements(INode node, int startOffset, int endOffset, boolean strict) {
    Set<FollowElement> result = Sets.newLinkedHashSet();
    TokenSource tokenSource = tokenSourceFactory.toTokenSource(node, startOffset, endOffset);
    CustomInternalN4JSParser parser = collectFollowElements(tokenSource, strict, result);
    adjustASIAndCollectFollowElements(parser, strict, result);
    /*
		 * Lists are easier to debug
		 */
    return Lists.newArrayList(result);
}

Example 15 with TokenSource

use of org.antlr.runtime.TokenSource in project xtext-eclipse by eclipse.

the class DocumentTokenSource method createTokenInfos.

protected List<TokenInfo> createTokenInfos(String string) {
    List<TokenInfo> result = Lists.newArrayListWithExpectedSize(string.length() / 3);
    TokenSource source = createTokenSource(string);
    CommonToken token = (CommonToken) source.nextToken();
    while (token != Token.EOF_TOKEN) {
        TokenInfo info = createTokenInfo(token);
        result.add(info);
        token = (CommonToken) source.nextToken();
    }
    return result;
}