Examples with XtextTokenStream org.eclipse.xtext.parser.antlr.XtextTokenStream used on opensource projects

Example 1 with XtextTokenStream

use of org.eclipse.xtext.parser.antlr.XtextTokenStream in project xtext-xtend by eclipse.

the class LexingTest method assertLexing.

protected void assertLexing(String input, Pair<String, String>... expectedTokens) {
    Lexer lexer = new InternalXtendLexer(null);
    CharStream stream = new ANTLRStringStream(input);
    lexer.setCharStream(stream);
    XtextTokenStream tokenStream = new XtextTokenStream(lexer, tokenDefProvider);
    List<?> tokens = tokenStream.getTokens();
    assertEquals(input + " / " + tokens, expectedTokens.length, tokens.size());
    for (int i = 0; i < tokens.size(); i++) {
        Token token = (Token) tokens.get(i);
        assertEquals(token.toString(), expectedTokens[i].getFirst(), token.getText());
        final String expected = expectedTokens[i].getSecond();
        String actual = tokenDefProvider.getTokenDefMap().get(token.getType());
        assertEquals("expected " + expected + " but was " + actual, expected, actual);
    }
}

Example 2 with XtextTokenStream

use of org.eclipse.xtext.parser.antlr.XtextTokenStream in project xtext-core by eclipse.

the class InternalBug292245TestLanguageParser method ruleModel.

// $ANTLR end "entryRuleModel"
// $ANTLR start "ruleModel"
// InternalBug292245TestLanguage.g:76:1: ruleModel returns [EObject current=null] : ( (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ ) | (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ ) | (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ ) )* ;
public final EObject ruleModel() throws RecognitionException {
    EObject current = null;
    Token otherlv_0 = null;
    Token otherlv_2 = null;
    Token otherlv_4 = null;
    AntlrDatatypeRuleToken lv_fix_1_0 = null;
    AntlrDatatypeRuleToken lv_error_3_0 = null;
    AntlrDatatypeRuleToken lv_tick_5_0 = null;
    enterRule();
    HiddenTokens myHiddenTokenState = ((XtextTokenStream) input).setHiddenTokens("RULE_WS");
    try {
        // InternalBug292245TestLanguage.g:83:2: ( ( (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ ) | (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ ) | (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ ) )* )
        // InternalBug292245TestLanguage.g:84:2: ( (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ ) | (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ ) | (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ ) )*
        {
            // InternalBug292245TestLanguage.g:84:2: ( (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ ) | (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ ) | (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ ) )*
            loop4: do {
                int alt4 = 4;
                switch(input.LA(1)) {
                    case 7:
                        {
                            alt4 = 1;
                        }
                        break;
                    case 8:
                        {
                            alt4 = 2;
                        }
                        break;
                    case 9:
                        {
                            alt4 = 3;
                        }
                        break;
                }
                switch(alt4) {
                    case 1:
                        // InternalBug292245TestLanguage.g:85:3: (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ )
                        {
                            // InternalBug292245TestLanguage.g:85:3: (otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+ )
                            // InternalBug292245TestLanguage.g:86:4: otherlv_0= 'FIX' ( (lv_fix_1_0= ruleFix ) )+
                            {
                                otherlv_0 = (Token) match(input, 7, FollowSets000.FOLLOW_3);
                                newLeafNode(otherlv_0, grammarAccess.getModelAccess().getFIXKeyword_0_0());
                                // InternalBug292245TestLanguage.g:90:4: ( (lv_fix_1_0= ruleFix ) )+
                                int cnt1 = 0;
                                loop1: do {
                                    int alt1 = 2;
                                    int LA1_0 = input.LA(1);
                                    if ((LA1_0 == RULE_APOSTROPHE_CHAR)) {
                                        alt1 = 1;
                                    }
                                    switch(alt1) {
                                        case 1:
                                            // InternalBug292245TestLanguage.g:91:5: (lv_fix_1_0= ruleFix )
                                            {
                                                // InternalBug292245TestLanguage.g:91:5: (lv_fix_1_0= ruleFix )
                                                // InternalBug292245TestLanguage.g:92:6: lv_fix_1_0= ruleFix
                                                {
                                                    newCompositeNode(grammarAccess.getModelAccess().getFixFixParserRuleCall_0_1_0());
                                                    pushFollow(FollowSets000.FOLLOW_4);
                                                    lv_fix_1_0 = ruleFix();
                                                    state._fsp--;
                                                    if (current == null) {
                                                        current = createModelElementForParent(grammarAccess.getModelRule());
                                                    }
                                                    add(current, "fix", lv_fix_1_0, "org.eclipse.xtext.parser.terminalrules.Bug292245TestLanguage.Fix");
                                                    afterParserOrEnumRuleCall();
                                                }
                                            }
                                            break;
                                        default:
                                            if (cnt1 >= 1)
                                                break loop1;
                                            EarlyExitException eee = new EarlyExitException(1, input);
                                            throw eee;
                                    }
                                    cnt1++;
                                } while (true);
                            }
                        }
                        break;
                    case 2:
                        // InternalBug292245TestLanguage.g:111:3: (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ )
                        {
                            // InternalBug292245TestLanguage.g:111:3: (otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+ )
                            // InternalBug292245TestLanguage.g:112:4: otherlv_2= 'ERROR' ( (lv_error_3_0= ruleError ) )+
                            {
                                otherlv_2 = (Token) match(input, 8, FollowSets000.FOLLOW_3);
                                newLeafNode(otherlv_2, grammarAccess.getModelAccess().getERRORKeyword_1_0());
                                // InternalBug292245TestLanguage.g:116:4: ( (lv_error_3_0= ruleError ) )+
                                int cnt2 = 0;
                                loop2: do {
                                    int alt2 = 2;
                                    int LA2_0 = input.LA(1);
                                    if ((LA2_0 == RULE_APOSTROPHE_CHAR)) {
                                        alt2 = 1;
                                    }
                                    switch(alt2) {
                                        case 1:
                                            // InternalBug292245TestLanguage.g:117:5: (lv_error_3_0= ruleError )
                                            {
                                                // InternalBug292245TestLanguage.g:117:5: (lv_error_3_0= ruleError )
                                                // InternalBug292245TestLanguage.g:118:6: lv_error_3_0= ruleError
                                                {
                                                    newCompositeNode(grammarAccess.getModelAccess().getErrorErrorParserRuleCall_1_1_0());
                                                    pushFollow(FollowSets000.FOLLOW_4);
                                                    lv_error_3_0 = ruleError();
                                                    state._fsp--;
                                                    if (current == null) {
                                                        current = createModelElementForParent(grammarAccess.getModelRule());
                                                    }
                                                    add(current, "error", lv_error_3_0, "org.eclipse.xtext.parser.terminalrules.Bug292245TestLanguage.Error");
                                                    afterParserOrEnumRuleCall();
                                                }
                                            }
                                            break;
                                        default:
                                            if (cnt2 >= 1)
                                                break loop2;
                                            EarlyExitException eee = new EarlyExitException(2, input);
                                            throw eee;
                                    }
                                    cnt2++;
                                } while (true);
                            }
                        }
                        break;
                    case 3:
                        // InternalBug292245TestLanguage.g:137:3: (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ )
                        {
                            // InternalBug292245TestLanguage.g:137:3: (otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+ )
                            // InternalBug292245TestLanguage.g:138:4: otherlv_4= 'TICK' ( (lv_tick_5_0= ruleApostrophe ) )+
                            {
                                otherlv_4 = (Token) match(input, 9, FollowSets000.FOLLOW_3);
                                newLeafNode(otherlv_4, grammarAccess.getModelAccess().getTICKKeyword_2_0());
                                // InternalBug292245TestLanguage.g:142:4: ( (lv_tick_5_0= ruleApostrophe ) )+
                                int cnt3 = 0;
                                loop3: do {
                                    int alt3 = 2;
                                    int LA3_0 = input.LA(1);
                                    if ((LA3_0 == RULE_APOSTROPHE_CHAR)) {
                                        alt3 = 1;
                                    }
                                    switch(alt3) {
                                        case 1:
                                            // InternalBug292245TestLanguage.g:143:5: (lv_tick_5_0= ruleApostrophe )
                                            {
                                                // InternalBug292245TestLanguage.g:143:5: (lv_tick_5_0= ruleApostrophe )
                                                // InternalBug292245TestLanguage.g:144:6: lv_tick_5_0= ruleApostrophe
                                                {
                                                    newCompositeNode(grammarAccess.getModelAccess().getTickApostropheParserRuleCall_2_1_0());
                                                    pushFollow(FollowSets000.FOLLOW_4);
                                                    lv_tick_5_0 = ruleApostrophe();
                                                    state._fsp--;
                                                    if (current == null) {
                                                        current = createModelElementForParent(grammarAccess.getModelRule());
                                                    }
                                                    add(current, "tick", lv_tick_5_0, "org.eclipse.xtext.parser.terminalrules.Bug292245TestLanguage.Apostrophe");
                                                    afterParserOrEnumRuleCall();
                                                }
                                            }
                                            break;
                                        default:
                                            if (cnt3 >= 1)
                                                break loop3;
                                            EarlyExitException eee = new EarlyExitException(3, input);
                                            throw eee;
                                    }
                                    cnt3++;
                                } while (true);
                            }
                        }
                        break;
                    default:
                        break loop4;
                }
            } while (true);
        }
        leaveRule();
    } catch (RecognitionException re) {
        recover(input, re);
        appendSkippedTokens();
    } finally {
        myHiddenTokenState.restore();
    }
    return current;
}

Example 3 with XtextTokenStream

use of org.eclipse.xtext.parser.antlr.XtextTokenStream in project xtext-core by eclipse.

the class InternalBug292245TestLanguageParser method entryRuleFix.

// $ANTLR end "ruleError"
// $ANTLR start "entryRuleFix"
// InternalBug292245TestLanguage.g:221:1: entryRuleFix returns [String current=null] : iv_ruleFix= ruleFix EOF ;
public final String entryRuleFix() throws RecognitionException {
    String current = null;
    AntlrDatatypeRuleToken iv_ruleFix = null;
    HiddenTokens myHiddenTokenState = ((XtextTokenStream) input).setHiddenTokens();
    try {
        // InternalBug292245TestLanguage.g:223:2: (iv_ruleFix= ruleFix EOF )
        // InternalBug292245TestLanguage.g:224:2: iv_ruleFix= ruleFix EOF
        {
            newCompositeNode(grammarAccess.getFixRule());
            pushFollow(FollowSets000.FOLLOW_1);
            iv_ruleFix = ruleFix();
            state._fsp--;
            current = iv_ruleFix.getText();
            match(input, EOF, FollowSets000.FOLLOW_2);
        }
    } catch (RecognitionException re) {
        recover(input, re);
        appendSkippedTokens();
    } finally {
        myHiddenTokenState.restore();
    }
    return current;
}

Example 4 with XtextTokenStream

use of org.eclipse.xtext.parser.antlr.XtextTokenStream in project xtext-core by eclipse.

the class InternalBug292245TestLanguageParser method ruleError.

// $ANTLR end "entryRuleError"
// $ANTLR start "ruleError"
// InternalBug292245TestLanguage.g:181:1: ruleError returns [AntlrDatatypeRuleToken current=new AntlrDatatypeRuleToken()] : (this_APOSTROPHE_CHAR_0= RULE_APOSTROPHE_CHAR this_Graphical_1= ruleGraphical this_APOSTROPHE_CHAR_2= RULE_APOSTROPHE_CHAR ) ;
public final AntlrDatatypeRuleToken ruleError() throws RecognitionException {
    AntlrDatatypeRuleToken current = new AntlrDatatypeRuleToken();
    Token this_APOSTROPHE_CHAR_0 = null;
    Token this_APOSTROPHE_CHAR_2 = null;
    AntlrDatatypeRuleToken this_Graphical_1 = null;
    enterRule();
    HiddenTokens myHiddenTokenState = ((XtextTokenStream) input).setHiddenTokens();
    try {
        // InternalBug292245TestLanguage.g:188:2: ( (this_APOSTROPHE_CHAR_0= RULE_APOSTROPHE_CHAR this_Graphical_1= ruleGraphical this_APOSTROPHE_CHAR_2= RULE_APOSTROPHE_CHAR ) )
        // InternalBug292245TestLanguage.g:189:2: (this_APOSTROPHE_CHAR_0= RULE_APOSTROPHE_CHAR this_Graphical_1= ruleGraphical this_APOSTROPHE_CHAR_2= RULE_APOSTROPHE_CHAR )
        {
            // InternalBug292245TestLanguage.g:189:2: (this_APOSTROPHE_CHAR_0= RULE_APOSTROPHE_CHAR this_Graphical_1= ruleGraphical this_APOSTROPHE_CHAR_2= RULE_APOSTROPHE_CHAR )
            // InternalBug292245TestLanguage.g:190:3: this_APOSTROPHE_CHAR_0= RULE_APOSTROPHE_CHAR this_Graphical_1= ruleGraphical this_APOSTROPHE_CHAR_2= RULE_APOSTROPHE_CHAR
            {
                this_APOSTROPHE_CHAR_0 = (Token) match(input, RULE_APOSTROPHE_CHAR, FollowSets000.FOLLOW_5);
                current.merge(this_APOSTROPHE_CHAR_0);
                newLeafNode(this_APOSTROPHE_CHAR_0, grammarAccess.getErrorAccess().getAPOSTROPHE_CHARTerminalRuleCall_0());
                newCompositeNode(grammarAccess.getErrorAccess().getGraphicalParserRuleCall_1());
                pushFollow(FollowSets000.FOLLOW_3);
                this_Graphical_1 = ruleGraphical();
                state._fsp--;
                current.merge(this_Graphical_1);
                afterParserOrEnumRuleCall();
                this_APOSTROPHE_CHAR_2 = (Token) match(input, RULE_APOSTROPHE_CHAR, FollowSets000.FOLLOW_2);
                current.merge(this_APOSTROPHE_CHAR_2);
                newLeafNode(this_APOSTROPHE_CHAR_2, grammarAccess.getErrorAccess().getAPOSTROPHE_CHARTerminalRuleCall_2());
            }
        }
        leaveRule();
    } catch (RecognitionException re) {
        recover(input, re);
        appendSkippedTokens();
    } finally {
        myHiddenTokenState.restore();
    }
    return current;
}

Example 5 with XtextTokenStream

use of org.eclipse.xtext.parser.antlr.XtextTokenStream in project xtext-core by eclipse.

the class InternalBug292245TestLanguageParser method entryRuleModel.

// $ANTLR start "entryRuleModel"
// InternalBug292245TestLanguage.g:64:1: entryRuleModel returns [EObject current=null] : iv_ruleModel= ruleModel EOF ;
public final EObject entryRuleModel() throws RecognitionException {
    EObject current = null;
    EObject iv_ruleModel = null;
    HiddenTokens myHiddenTokenState = ((XtextTokenStream) input).setHiddenTokens("RULE_WS");
    try {
        // InternalBug292245TestLanguage.g:66:2: (iv_ruleModel= ruleModel EOF )
        // InternalBug292245TestLanguage.g:67:2: iv_ruleModel= ruleModel EOF
        {
            newCompositeNode(grammarAccess.getModelRule());
            pushFollow(FollowSets000.FOLLOW_1);
            iv_ruleModel = ruleModel();
            state._fsp--;
            current = iv_ruleModel;
            match(input, EOF, FollowSets000.FOLLOW_2);
        }
    } catch (RecognitionException re) {
        recover(input, re);
        appendSkippedTokens();
    } finally {
        myHiddenTokenState.restore();
    }
    return current;
}