Examples with MatchToken org.antlr.v4.codegen.model.MatchToken used on opensource projects

Example 1 with MatchToken

use of org.antlr.v4.codegen.model.MatchToken in project antlr4 by tunnelvisionlabs.

the class ParserFactory method tokenRef.

@Override
public List<SrcOp> tokenRef(GrammarAST ID, GrammarAST labelAST, GrammarAST args) {
    MatchToken matchOp = new MatchToken(this, (TerminalAST) ID);
    if (labelAST != null) {
        String label = labelAST.getText();
        RuleFunction rf = getCurrentRuleFunction();
        if (labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN) {
            // add Token _X and List<Token> X decls
            // adds _X
            defineImplicitLabel(ID, matchOp);
            TokenListDecl l = getTokenListLabelDecl(label);
            rf.addContextDecl(ID.getAltLabel(), l);
        } else {
            Decl d = getTokenLabelDecl(label);
            matchOp.labels.add(d);
            rf.addContextDecl(ID.getAltLabel(), d);
        }
    // Decl d = getTokenLabelDecl(label);
    // ((MatchToken)matchOp).labels.add(d);
    // getCurrentRuleFunction().addContextDecl(ID.getAltLabel(), d);
    // if ( labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN ) {
    // TokenListDecl l = getTokenListLabelDecl(label);
    // getCurrentRuleFunction().addContextDecl(ID.getAltLabel(), l);
    // }
    }
    if (controller.needsImplicitLabel(ID, matchOp))
        defineImplicitLabel(ID, matchOp);
    AddToLabelList listLabelOp = getAddToListOpIfListLabelPresent(matchOp, labelAST);
    return list(matchOp, listLabelOp);
}

Example 2 with MatchToken

use of org.antlr.v4.codegen.model.MatchToken in project antlr4 by antlr.

the class ParserFactory method tokenRef.

@Override
public List<SrcOp> tokenRef(GrammarAST ID, GrammarAST labelAST, GrammarAST args) {
    MatchToken matchOp = new MatchToken(this, (TerminalAST) ID);
    if (labelAST != null) {
        String label = labelAST.getText();
        RuleFunction rf = getCurrentRuleFunction();
        if (labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN) {
            // add Token _X and List<Token> X decls
            // adds _X
            defineImplicitLabel(ID, matchOp);
            TokenListDecl l = getTokenListLabelDecl(label);
            rf.addContextDecl(ID.getAltLabel(), l);
        } else {
            Decl d = getTokenLabelDecl(label);
            matchOp.labels.add(d);
            rf.addContextDecl(ID.getAltLabel(), d);
        }
    // Decl d = getTokenLabelDecl(label);
    // ((MatchToken)matchOp).labels.add(d);
    // getCurrentRuleFunction().addContextDecl(ID.getAltLabel(), d);
    // if ( labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN ) {
    // TokenListDecl l = getTokenListLabelDecl(label);
    // getCurrentRuleFunction().addContextDecl(ID.getAltLabel(), l);
    // }
    }
    if (controller.needsImplicitLabel(ID, matchOp))
        defineImplicitLabel(ID, matchOp);
    AddToLabelList listLabelOp = getAddToListOpIfListLabelPresent(matchOp, labelAST);
    return list(matchOp, listLabelOp);
}

Example 3 with MatchToken

use of org.antlr.v4.codegen.model.MatchToken in project zemberek-nlp by ahmetaa.

the class TurkishTokenizerTest method matchToken.

private void matchToken(TurkishTokenizer tokenizer, String input, int tokenType, String... expectedTokens) {
    List<Token> tokens = tokenizer.tokenize(input);
    Assert.assertNotNull("Token list is null.", tokens);
    Assert.assertTrue(tokens.size() > 0);
    Assert.assertEquals("Token count is not equal to expected Token count for input " + input, expectedTokens.length, tokens.size());
    int i = 0;
    for (String expectedToken : expectedTokens) {
        Token token = tokens.get(i);
        Assert.assertEquals(expectedToken + " is not equal to " + token.getText(), expectedToken, token.getText());
        if (tokenType != -1) {
            Assert.assertEquals(tokenType, token.getType());
        }
        i++;
    }
}