Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 71 with Token

use of org.antlr.v4.runtime.Token in project elasticsearch by elastic.

the class ParserErrorStrategy method recoverInline.

@Override
public Token recoverInline(final Parser recognizer) throws RecognitionException {
    final Token token = recognizer.getCurrentToken();
    final String message = "unexpected token [" + getTokenErrorDisplay(token) + "]" + " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
    Location location = new Location(sourceName, token.getStartIndex());
    throw location.createError(new IllegalArgumentException(message));
}

Example 72 with Token

use of org.antlr.v4.runtime.Token in project dex2jar by pxb1988.

the class AntlrSmaliUtil method acceptField.

public static void acceptField(SmaliParser.SFieldContext ctx, String className, DexClassVisitor dexClassVisitor) {
    Field field;
    Token fieldObj = ctx.fieldObj;
    if (fieldObj.getType() == SmaliLexer.FIELD_FULL) {
        field = Utils.parseFieldAndUnescape(fieldObj.getText());
    } else {
        field = Utils.parseFieldAndUnescape(className, fieldObj.getText());
    }
    int access = collectAccess(ctx.sAccList());
    Object value = null;
    SmaliParser.SBaseValueContext vctx = ctx.sBaseValue();
    if (vctx != null) {
        value = parseBaseValue(vctx);
    }
    DexFieldVisitor dexFieldVisitor = dexClassVisitor.visitField(access, field, value);
    if (dexFieldVisitor != null) {
        acceptAnnotations(ctx.sAnnotation(), dexFieldVisitor);
        dexFieldVisitor.visitEnd();
    }
}

Example 73 with Token

use of org.antlr.v4.runtime.Token in project jetbrick-template-1x by subchen.

the class JetTemplateCodeVisitor method visitConstant.

@Override
public Code visitConstant(ConstantContext ctx) {
    Token token = ((TerminalNode) ctx.getChild(0)).getSymbol();
    String text = token.getText();
    switch(token.getType()) {
        case JetTemplateParser.STRING_DOUBLE:
            return new SegmentCode(String.class, text, ctx);
        case JetTemplateParser.STRING_SINGLE:
            text = StringEscapeUtils.asCanonicalJavaString(text);
            return new SegmentCode(String.class, text, ctx);
        case JetTemplateParser.INTEGER:
        case JetTemplateParser.INTEGER_HEX:
        case JetTemplateParser.FLOATING_POINT:
            Class<?> klass;
            if (text.endsWith("l") || text.endsWith("L")) {
                klass = Long.TYPE;
            } else if (text.endsWith("f") || text.endsWith("F")) {
                klass = Float.TYPE;
            } else if (text.endsWith("d") || text.endsWith("D")) {
                klass = Double.TYPE;
            } else if (token.getType() == JetTemplateParser.FLOATING_POINT) {
                // 浮点数默认是double
                klass = Double.TYPE;
            } else {
                klass = Integer.TYPE;
            }
            return new SegmentCode(klass, text, ctx);
        case JetTemplateParser.KEYWORD_TRUE:
            return new SegmentCode(Boolean.TYPE, text, ctx);
        case JetTemplateParser.KEYWORD_FALSE:
            return new SegmentCode(Boolean.TYPE, text, ctx);
        case JetTemplateParser.KEYWORD_NULL:
            return new SegmentCode(TypedKlass.NULL, text, ctx);
        default:
            throw reportError("Unexpected token type :" + token.getType(), ctx);
    }
}

Example 74 with Token

use of org.antlr.v4.runtime.Token in project sqldelight by square.

the class PsiTokenSource method nextToken.

/* Colin: "the parsing lexer still has to return tokens that completely
	 cover the file (i.e. no gaps). This is one of the most significant
	 differences from a traditional compiler parser/lexer."
	  after lots of trial and error I finally just put the BAD_TOKEN
	  into the white space class so that they do not come to the parser
	  but that IDEA still knows about them.
	  parrt: this seems no longer to be true after Sam's re-factoring
	 */
@Override
public Token nextToken() {
    TokenElementType ideaTType = (TokenElementType) builder.getTokenType();
    int type;
    if (ideaTType == null) {
        type = Token.EOF;
    } else {
        type = ideaTType.getType();
    }
    int channel = Token.DEFAULT_CHANNEL;
    Pair<TokenSource, CharStream> source = new Pair<TokenSource, CharStream>(this, null);
    String text = builder.getTokenText();
    int start = builder.getCurrentOffset();
    int length = text != null ? text.length() : 0;
    int stop = start + length - 1;
    // PsiBuilder doesn't provide line, column info
    int line = 0;
    int charPositionInLine = 0;
    Token t = factory.create(source, type, text, channel, start, stop, line, charPositionInLine);
    builder.advanceLexer();
    //		System.out.println("TOKEN: "+t);
    return t;
}

Example 75 with Token

use of org.antlr.v4.runtime.Token in project presto by prestodb.

the class StatementSplitter method squeezeStatement.

public static String squeezeStatement(String sql) {
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        } else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}