Examples with Tuple2 org.antlr.v4.runtime.misc.Tuple2 used on opensource projects

Example 6 with Tuple2

use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.

the class ScopeParser method parseAttributeDef.

/**
 * For decls like "String foo" or "char *foo32[]" compute the ID
 * and type declarations.  Also handle "int x=3" and 'T t = new T("foo")'
 * but if the separator is ',' you cannot use ',' in the initvalue
 * unless you escape use "\," escape.
 */
public static Attribute parseAttributeDef(@Nullable ActionAST action, @NotNull Tuple2<String, Integer> decl, Grammar g) {
    if (decl.getItem1() == null)
        return null;
    Attribute attr = new Attribute();
    int rightEdgeOfDeclarator = decl.getItem1().length() - 1;
    int equalsIndex = decl.getItem1().indexOf('=');
    if (equalsIndex > 0) {
        // everything after the '=' is the init value
        attr.initValue = decl.getItem1().substring(equalsIndex + 1, decl.getItem1().length()).trim();
        rightEdgeOfDeclarator = equalsIndex - 1;
    }
    String declarator = decl.getItem1().substring(0, rightEdgeOfDeclarator + 1);
    Tuple2<Integer, Integer> p;
    String text = decl.getItem1();
    text = text.replaceAll("::", "");
    if (text.contains(":")) {
        // declarator has type appearing after the name like "x:T"
        p = _parsePostfixDecl(attr, declarator, action, g);
    } else {
        // declarator has type appearing before the name like "T x"
        p = _parsePrefixDecl(attr, declarator, action, g);
    }
    int idStart = p.getItem1();
    int idStop = p.getItem2();
    attr.decl = decl.getItem1();
    if (action != null) {
        String actionText = action.getText();
        int[] lines = new int[actionText.length()];
        int[] charPositionInLines = new int[actionText.length()];
        for (int i = 0, line = 0, col = 0; i < actionText.length(); i++, col++) {
            lines[i] = line;
            charPositionInLines[i] = col;
            if (actionText.charAt(i) == '\n') {
                line++;
                col = -1;
            }
        }
        int[] charIndexes = new int[actionText.length()];
        for (int i = 0, j = 0; i < actionText.length(); i++, j++) {
            charIndexes[j] = i;
            // skip comments
            if (i < actionText.length() - 1 && actionText.charAt(i) == '/' && actionText.charAt(i + 1) == '/') {
                while (i < actionText.length() && actionText.charAt(i) != '\n') {
                    i++;
                }
            }
        }
        int declOffset = charIndexes[decl.getItem2()];
        int declLine = lines[declOffset + idStart];
        int line = action.getToken().getLine() + declLine;
        int charPositionInLine = charPositionInLines[declOffset + idStart];
        if (declLine == 0) {
            /* offset for the start position of the ARG_ACTION token, plus 1
				 * since the ARG_ACTION text had the leading '[' stripped before
				 * reaching the scope parser.
				 */
            charPositionInLine += action.getToken().getCharPositionInLine() + 1;
        }
        int offset = ((CommonToken) action.getToken()).getStartIndex();
        attr.token = new CommonToken(action.getToken().getInputStream(), ANTLRParser.ID, BaseRecognizer.DEFAULT_TOKEN_CHANNEL, offset + declOffset + idStart + 1, offset + declOffset + idStop);
        attr.token.setLine(line);
        attr.token.setCharPositionInLine(charPositionInLine);
        assert attr.name.equals(attr.token.getText()) : "Attribute text should match the pseudo-token text at this point.";
    }
    return attr;
}

Example 7 with Tuple2

use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.

the class TestXPath method testError.

protected void testError(String input, String path, String expected, String startRuleName, String parserName, String lexerName) throws Exception {
    Tuple2<Parser, Lexer> pl = getParserAndLexer(input, parserName, lexerName);
    Parser parser = pl.getItem1();
    ParseTree tree = execStartRule(startRuleName, parser);
    IllegalArgumentException e = null;
    try {
        XPath.findAll(tree, path, parser);
    } catch (IllegalArgumentException iae) {
        e = iae;
    }
    assertNotNull(e);
    assertEquals(expected, e.getMessage());
}

Example 8 with Tuple2

use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.

the class Grammar method getStringLiteralAliasesFromLexerRules.

/**
 * Return list of (TOKEN_NAME node, 'literal' node) pairs
 */
public static List<Tuple2<GrammarAST, GrammarAST>> getStringLiteralAliasesFromLexerRules(GrammarRootAST ast) {
    String[] patterns = { "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL)))", "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL ACTION)))", "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL SEMPRED)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) .)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) . .)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) (LEXER_ACTION_CALL . .))))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) . (LEXER_ACTION_CALL . .))))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) (LEXER_ACTION_CALL . .) .)))" // TODO: allow doc comment in there
    };
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(ast.token.getInputStream());
    org.antlr.runtime.tree.TreeWizard wiz = new org.antlr.runtime.tree.TreeWizard(adaptor, ANTLRParser.tokenNames);
    List<Tuple2<GrammarAST, GrammarAST>> lexerRuleToStringLiteral = new ArrayList<Tuple2<GrammarAST, GrammarAST>>();
    List<GrammarAST> ruleNodes = ast.getNodesWithType(ANTLRParser.RULE);
    if (ruleNodes == null || ruleNodes.isEmpty())
        return null;
    for (GrammarAST r : ruleNodes) {
        // tool.log("grammar", r.toStringTree());
        // System.out.println("chk: "+r.toStringTree());
        org.antlr.runtime.tree.Tree name = r.getChild(0);
        if (name.getType() == ANTLRParser.TOKEN_REF) {
            // check rule against patterns
            boolean isLitRule;
            for (String pattern : patterns) {
                isLitRule = defAlias(r, pattern, wiz, lexerRuleToStringLiteral);
                if (isLitRule)
                    break;
            }
        // if ( !isLitRule ) System.out.println("no pattern matched");
        }
    }
    return lexerRuleToStringLiteral;
}

Example 9 with Tuple2

use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.

the class GrammarTransformPipeline method extractImplicitLexer.

/**
 * Build lexer grammar from combined grammar that looks like:
 *
 *  (COMBINED_GRAMMAR A
 *      (tokens { X (= Y 'y'))
 *      (OPTIONS (= x 'y'))
 *      (@ members {foo})
 *      (@ lexer header {package jj;})
 *      (RULES (RULE .+)))
 *
 *  Move rules and actions to new tree, don't dup. Split AST apart.
 *  We'll have this Grammar share token symbols later; don't generate
 *  tokenVocab or tokens{} section.  Copy over named actions.
 *
 *  Side-effects: it removes children from GRAMMAR & RULES nodes
 *                in combined AST.  Anything cut out is dup'd before
 *                adding to lexer to avoid "who's ur daddy" issues
 */
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
    GrammarRootAST combinedAST = combinedGrammar.ast;
    // tool.log("grammar", "before="+combinedAST.toStringTree());
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
    GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
    // MAKE A GRAMMAR ROOT and ID
    String lexerName = combinedAST.getChild(0).getText() + "Lexer";
    GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
    lexerAST.grammarType = ANTLRParser.LEXER;
    lexerAST.token.setInputStream(combinedAST.token.getInputStream());
    lexerAST.addChild(adaptor.create(ANTLRParser.ID, lexerName));
    // COPY OPTIONS
    GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
    if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
        GrammarAST lexerOptionsRoot = adaptor.dupNode(optionsRoot);
        lexerAST.addChild(lexerOptionsRoot);
        GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
        for (GrammarAST o : options) {
            String optionName = o.getChild(0).getText();
            if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
                GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
                lexerOptionsRoot.addChild(optionTree);
                lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
            }
        }
    }
    // COPY all named actions, but only move those with lexer:: scope
    List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
    for (GrammarAST e : elements) {
        if (e.getType() == ANTLRParser.AT) {
            lexerAST.addChild((Tree) adaptor.dupTree(e));
            if (e.getChild(0).getText().equals("lexer")) {
                actionsWeMoved.add(e);
            }
        }
    }
    for (GrammarAST r : actionsWeMoved) {
        combinedAST.deleteChild(r);
    }
    GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
    if (combinedRulesRoot == null)
        return lexerAST;
    // MOVE lexer rules
    GrammarAST lexerRulesRoot = adaptor.create(ANTLRParser.RULES, "RULES");
    lexerAST.addChild(lexerRulesRoot);
    List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
    GrammarASTWithOptions[] rules;
    if (combinedRulesRoot.getChildCount() > 0) {
        rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
    } else {
        rules = new GrammarASTWithOptions[0];
    }
    for (GrammarASTWithOptions r : rules) {
        String ruleName = r.getChild(0).getText();
        if (Grammar.isTokenName(ruleName)) {
            lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
            rulesWeMoved.add(r);
        }
    }
    for (GrammarAST r : rulesWeMoved) {
        combinedRulesRoot.deleteChild(r);
    }
    // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
    List<Tuple2<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
    Set<String> stringLiterals = combinedGrammar.getStringLiterals();
    // add strings from combined grammar (and imported grammars) into lexer
    // put them first as they are keywords; must resolve ambigs to these rules
    // tool.log("grammar", "strings from parser: "+stringLiterals);
    int insertIndex = 0;
    nextLit: for (String lit : stringLiterals) {
        // if lexer already has a rule for literal, continue
        if (litAliases != null) {
            for (Tuple2<GrammarAST, GrammarAST> pair : litAliases) {
                GrammarAST litAST = pair.getItem2();
                if (lit.equals(litAST.getText()))
                    continue nextLit;
            }
        }
        // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
        String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
        // can't use wizard; need special node types
        GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
        BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
        AltAST alt = new AltAST(ANTLRParser.ALT);
        TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
        alt.addChild(slit);
        blk.addChild(alt);
        CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
        litRule.addChild(new TerminalAST(idToken));
        litRule.addChild(blk);
        lexerRulesRoot.insertChild(insertIndex, litRule);
        // lexerRulesRoot.getChildren().add(0, litRule);
        // reset indexes and set litRule parent
        lexerRulesRoot.freshenParentAndChildIndexes();
        // next literal will be added after the one just added
        insertIndex++;
    }
    // TODO: take out after stable if slow
    lexerAST.sanityCheckParentAndChildIndexes();
    combinedAST.sanityCheckParentAndChildIndexes();
    // tool.log("grammar", combinedAST.toTokenString());
    combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
    combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
    if (lexerRulesRoot.getChildCount() == 0)
        return null;
    return lexerAST;
}

Example 10 with Tuple2

use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.

the class BaseTest method execParser.

public ParseTree execParser(String startRuleName, String input, String parserName, String lexerName) throws Exception {
    Tuple2<Parser, Lexer> pl = getParserAndLexer(input, parserName, lexerName);
    Parser parser = pl.getItem1();
    return execStartRule(startRuleName, parser);
}