Examples with IToken org.eclipse.jface.text.rules.IToken used on opensource projects

Example 31 with IToken

use of org.eclipse.jface.text.rules.IToken in project erlide_eclipse by erlang.

the class ErlangCharRuleTest method hex_6.

@Test
public void hex_6() {
    final ErlangCharRule rule = new ErlangCharRule(token);
    scanner = new StringCharacterScanner("$\\x1");
    final IToken tk = rule.evaluate(scanner);
    assertEquals(token, tk);
    assertEquals("", scanner.getCurrentContent());
    assertEquals("$\\x1", scanner.getScannedContent());
}

Example 32 with IToken

use of org.eclipse.jface.text.rules.IToken in project erlide_eclipse by erlang.

the class ErlangCharRuleTest method hex_2.

@Test
public void hex_2() {
    final ErlangCharRule rule = new ErlangCharRule(token);
    scanner = new StringCharacterScanner("$\\x{cFE1}23z ");
    final IToken tk = rule.evaluate(scanner);
    assertEquals(token, tk);
    assertEquals("23z ", scanner.getCurrentContent());
    assertEquals("$\\x{cFE1}", scanner.getScannedContent());
}

Example 33 with IToken

use of org.eclipse.jface.text.rules.IToken in project erlide_eclipse by erlang.

the class ErlangCharRuleTest method hex_5.

@Test
public void hex_5() {
    final ErlangCharRule rule = new ErlangCharRule(token);
    scanner = new StringCharacterScanner("$\\x12");
    final IToken tk = rule.evaluate(scanner);
    assertEquals(token, tk);
    assertEquals("", scanner.getCurrentContent());
    assertEquals("$\\x12", scanner.getScannedContent());
}

Example 34 with IToken

use of org.eclipse.jface.text.rules.IToken in project erlide_eclipse by erlang.

the class ErlangCharRuleTest method simpleChar_1.

@Test
public void simpleChar_1() {
    final ErlangCharRule rule = new ErlangCharRule(token);
    scanner = new StringCharacterScanner("$z");
    final IToken tk = rule.evaluate(scanner);
    assertEquals(token, tk);
    assertEquals("", scanner.getCurrentContent());
    assertEquals("$z", scanner.getScannedContent());
}

Example 35 with IToken

use of org.eclipse.jface.text.rules.IToken in project xtext-eclipse by eclipse.

the class DocumentPartitioner method initialize.

/**
 * Performs the initial partitioning of the partitioner's document.
 * <p>
 * May be extended by subclasses.
 * </p>
 *
 * @since 2.2
 */
protected void initialize() {
    fIsInitialized = true;
    clearPositionCache();
    fScanner.setRange(fDocument, 0, fDocument.getLength());
    try {
        IToken token = fScanner.nextToken();
        while (!token.isEOF()) {
            String contentType = getTokenContentType(token);
            if (isSupportedContentType(contentType)) {
                TypedPosition p = new TypedPosition(fScanner.getTokenOffset(), fScanner.getTokenLength(), contentType);
                fDocument.addPosition(fPositionCategory, p);
            }
            token = fScanner.nextToken();
        }
    } catch (BadLocationException x) {
    // cannot happen as offsets come from scanner
    } catch (BadPositionCategoryException x) {
    // cannot happen if document has been connected before
    }
}