Examples with Token antlr.Token used on opensource projects

Example 1 with Token

use of antlr.Token in project groovy by apache.

the class LexerFrame method scanScript.

private void scanScript(final StringReader reader) throws Exception {
    scriptPane.read(reader, null);
    reader.reset();
    // create lexer
    final Constructor constructor = lexerClass.getConstructor(Reader.class);
    final CharScanner lexer = (CharScanner) constructor.newInstance(reader);
    tokenPane.setEditable(true);
    tokenPane.setText("");
    int line = 1;
    final ButtonGroup bg = new ButtonGroup();
    Token token;
    while (true) {
        token = lexer.nextToken();
        JToggleButton tokenButton = new JToggleButton((String) tokens.get(Integer.valueOf(token.getType())));
        bg.add(tokenButton);
        tokenButton.addActionListener(this);
        tokenButton.setToolTipText(token.getText());
        tokenButton.putClientProperty("token", token);
        tokenButton.setMargin(new Insets(0, 1, 0, 1));
        tokenButton.setFocusPainted(false);
        if (token.getLine() > line) {
            tokenPane.getDocument().insertString(tokenPane.getDocument().getLength(), "\n", null);
            line = token.getLine();
        }
        insertComponent(tokenButton);
        if (token.getType() == Token.EOF_TYPE) {
            break;
        }
    }
    tokenPane.setEditable(false);
    tokenPane.setCaretPosition(0);
    reader.close();
}

Example 2 with Token

use of antlr.Token in project groovy by apache.

the class Main method parseFile.

// Here's where we do the real work...
public static void parseFile(String f, GroovyLexer l, SourceBuffer sourceBuffer) throws Exception {
    try {
        // Create a parser that reads from the scanner
        GroovyRecognizer parser = GroovyRecognizer.make(l);
        parser.setSourceBuffer(sourceBuffer);
        parser.setFilename(f);
        if (whitespaceIncluded) {
            GroovyLexer lexer = parser.getLexer();
            lexer.setWhitespaceIncluded(true);
            while (true) {
                Token t = lexer.nextToken();
                System.out.println(t);
                if (t == null || t.getType() == Token.EOF_TYPE)
                    break;
            }
            return;
        }
        // start parsing at the compilationUnit rule
        parser.compilationUnit();
        System.out.println("parseFile " + f + " => " + parser.getAST());
        // do something with the tree
        doTreeAction(f, parser.getAST(), parser.getTokenNames());
    } catch (Exception e) {
        System.err.println("parser exception: " + e);
        // so we can get stack trace        
        e.printStackTrace();
    }
}

Example 3 with Token

use of antlr.Token in project sonarqube by SonarSource.

the class ValidWhenLexer method mOCTAL_LITERAL.

public final void mOCTAL_LITERAL(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    int _ttype;
    Token _token = null;
    int _begin = text.length();
    _ttype = OCTAL_LITERAL;
    int _saveIndex;
    match('0');
    {
        _loop26: do {
            if (((LA(1) >= '0' && LA(1) <= '7'))) {
                matchRange('0', '7');
            } else {
                break _loop26;
            }
        } while (true);
    }
    if (_createToken && _token == null && _ttype != Token.SKIP) {
        _token = makeToken(_ttype);
        _token.setText(new String(text.getBuffer(), _begin, text.length() - _begin));
    }
    _returnToken = _token;
}

Example 4 with Token

use of antlr.Token in project sonarqube by SonarSource.

the class ValidWhenLexer method mRPAREN.

public final void mRPAREN(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    int _ttype;
    Token _token = null;
    int _begin = text.length();
    _ttype = RPAREN;
    int _saveIndex;
    match(')');
    if (_createToken && _token == null && _ttype != Token.SKIP) {
        _token = makeToken(_ttype);
        _token.setText(new String(text.getBuffer(), _begin, text.length() - _begin));
    }
    _returnToken = _token;
}

Example 5 with Token

use of antlr.Token in project sonarqube by SonarSource.

the class ValidWhenLexer method mDECIMAL_LITERAL.

public final void mDECIMAL_LITERAL(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    int _ttype;
    Token _token = null;
    int _begin = text.length();
    _ttype = DECIMAL_LITERAL;
    int _saveIndex;
    {
        switch(LA(1)) {
            case '-':
                {
                    match('-');
                    break;
                }
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                {
                    break;
                }
            default:
                {
                    throw new NoViableAltForCharException((char) LA(1), getFilename(), getLine(), getColumn());
                }
        }
    }
    {
        matchRange('1', '9');
    }
    {
        _loop20: do {
            if (((LA(1) >= '0' && LA(1) <= '9'))) {
                matchRange('0', '9');
            } else {
                break _loop20;
            }
        } while (true);
    }
    if (_createToken && _token == null && _ttype != Token.SKIP) {
        _token = makeToken(_ttype);
        _token.setText(new String(text.getBuffer(), _begin, text.length() - _begin));
    }
    _returnToken = _token;
}