Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 11 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ParseTreePatternMatcher method compile.

/**
	 * For repeated use of a tree pattern, compile it to a
	 * {@link ParseTreePattern} using this method.
	 */
public ParseTreePattern compile(String pattern, int patternRuleIndex) {
    List<? extends Token> tokenList = tokenize(pattern);
    ListTokenSource tokenSrc = new ListTokenSource(tokenList);
    CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
    ParserInterpreter parserInterp = new ParserInterpreter(parser.getGrammarFileName(), parser.getVocabulary(), Arrays.asList(parser.getRuleNames()), parser.getATNWithBypassAlts(), tokens);
    ParseTree tree = null;
    try {
        parserInterp.setErrorHandler(new BailErrorStrategy());
        tree = parserInterp.parse(patternRuleIndex);
    //			System.out.println("pattern tree = "+tree.toStringTree(parserInterp));
    } catch (ParseCancellationException e) {
        throw (RecognitionException) e.getCause();
    } catch (RecognitionException re) {
        throw re;
    } catch (Exception e) {
        throw new CannotInvokeStartRule(e);
    }
    // Make sure tree pattern compilation checks for a complete parse
    if (tokens.LA(1) != Token.EOF) {
        throw new StartRuleDoesNotConsumeFullPattern();
    }
    return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
}

Example 12 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class XPath method split.

// TODO: check for invalid token/rule names, bad syntax
public XPathElement[] split(String path) {
    ANTLRInputStream in;
    try {
        in = new ANTLRInputStream(new StringReader(path));
    } catch (IOException ioe) {
        throw new IllegalArgumentException("Could not read path: " + path, ioe);
    }
    XPathLexer lexer = new XPathLexer(in) {

        @Override
        public void recover(LexerNoViableAltException e) {
            throw e;
        }
    };
    lexer.removeErrorListeners();
    lexer.addErrorListener(new XPathLexerErrorListener());
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    try {
        tokenStream.fill();
    } catch (LexerNoViableAltException e) {
        int pos = lexer.getCharPositionInLine();
        String msg = "Invalid tokens or characters at index " + pos + " in path '" + path + "'";
        throw new IllegalArgumentException(msg, e);
    }
    List<Token> tokens = tokenStream.getTokens();
    //		System.out.println("path="+path+"=>"+tokens);
    List<XPathElement> elements = new ArrayList<XPathElement>();
    int n = tokens.size();
    int i = 0;
    loop: while (i < n) {
        Token el = tokens.get(i);
        Token next = null;
        switch(el.getType()) {
            case XPathLexer.ROOT:
            case XPathLexer.ANYWHERE:
                boolean anywhere = el.getType() == XPathLexer.ANYWHERE;
                i++;
                next = tokens.get(i);
                boolean invert = next.getType() == XPathLexer.BANG;
                if (invert) {
                    i++;
                    next = tokens.get(i);
                }
                XPathElement pathElement = getXPathElement(next, anywhere);
                pathElement.invert = invert;
                elements.add(pathElement);
                i++;
                break;
            case XPathLexer.TOKEN_REF:
            case XPathLexer.RULE_REF:
            case XPathLexer.WILDCARD:
                elements.add(getXPathElement(el, false));
                i++;
                break;
            case Token.EOF:
                break loop;
            default:
                throw new IllegalArgumentException("Unknowth path element " + el);
        }
    }
    return elements.toArray(new XPathElement[0]);
}

Example 13 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class XPath method evaluate.

/**
	 * Return a list of all nodes starting at {@code t} as root that satisfy the
	 * path. The root {@code /} is relative to the node passed to
	 * {@link #evaluate}.
	 */
public Collection<ParseTree> evaluate(final ParseTree t) {
    ParserRuleContext dummyRoot = new ParserRuleContext();
    // don't set t's parent.
    dummyRoot.children = Collections.singletonList(t);
    Collection<ParseTree> work = Collections.<ParseTree>singleton(dummyRoot);
    int i = 0;
    while (i < elements.length) {
        Collection<ParseTree> next = new LinkedHashSet<ParseTree>();
        for (ParseTree node : work) {
            if (node.getChildCount() > 0) {
                // only try to match next element if it has children
                // e.g., //func/*/stat might have a token node for which
                // we can't go looking for stat nodes.
                Collection<? extends ParseTree> matching = elements[i].evaluate(node);
                next.addAll(matching);
            }
        }
        i++;
        work = next;
    }
    return work;
}

Example 14 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class XPathLexer method nextToken.

@Override
public Token nextToken() {
    _tokenStartCharIndex = _input.index();
    CommonToken t = null;
    while (t == null) {
        switch(_input.LA(1)) {
            case '/':
                consume();
                if (_input.LA(1) == '/') {
                    consume();
                    t = new CommonToken(ANYWHERE, "//");
                } else {
                    t = new CommonToken(ROOT, "/");
                }
                break;
            case '*':
                consume();
                t = new CommonToken(WILDCARD, "*");
                break;
            case '!':
                consume();
                t = new CommonToken(BANG, "!");
                break;
            case '\'':
                String s = matchString();
                t = new CommonToken(STRING, s);
                break;
            case CharStream.EOF:
                return new CommonToken(EOF, "<EOF>");
            default:
                if (isNameStartChar(_input.LA(1))) {
                    String id = matchID();
                    if (Character.isUpperCase(id.charAt(0)))
                        t = new CommonToken(TOKEN_REF, id);
                    else
                        t = new CommonToken(RULE_REF, id);
                } else {
                    throw new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null);
                }
                break;
        }
    }
    t.setStartIndex(_tokenStartCharIndex);
    t.setCharPositionInLine(_tokenStartCharIndex);
    t.setLine(line);
    return t;
}

Example 15 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class DefaultErrorStrategy method reportUnwantedToken.

/**
	 * This method is called to report a syntax error which requires the removal
	 * of a token from the input stream. At the time this method is called, the
	 * erroneous symbol is current {@code LT(1)} symbol and has not yet been
	 * removed from the input stream. When this method returns,
	 * {@code recognizer} is in error recovery mode.
	 *
	 * <p>This method is called when {@link #singleTokenDeletion} identifies
	 * single-token deletion as a viable recovery strategy for a mismatched
	 * input error.</p>
	 *
	 * <p>The default implementation simply returns if the handler is already in
	 * error recovery mode. Otherwise, it calls {@link #beginErrorCondition} to
	 * enter error recovery mode, followed by calling
	 * {@link Parser#notifyErrorListeners}.</p>
	 *
	 * @param recognizer the parser instance
	 */
protected void reportUnwantedToken(Parser recognizer) {
    if (inErrorRecoveryMode(recognizer)) {
        return;
    }
    beginErrorCondition(recognizer);
    Token t = recognizer.getCurrentToken();
    String tokenName = getTokenErrorDisplay(t);
    IntervalSet expecting = getExpectedTokens(recognizer);
    String msg = "extraneous input " + tokenName + " expecting " + expecting.toString(recognizer.getVocabulary());
    recognizer.notifyErrorListeners(t, msg, null);
}