Examples with Token com.twitter.common.text.token.TokenizedCharSequence.Token used on opensource projects

Example 1 with Token

use of com.twitter.common.text.token.TokenizedCharSequence.Token in project commons by twitter.

the class TokenizerUsageExample method main.

public static void main(String[] args) {
    // This is the canonical way to create a token stream.
    DefaultTextTokenizer tokenizer = new DefaultTextTokenizer.Builder().setKeepPunctuation(true).build();
    TwitterTokenStream stream = tokenizer.getDefaultTokenStream();
    // We're going to ask the token stream what type of attributes it makes available. "Attributes"
    // can be understood as "annotations" on the original text.
    System.out.println("Attributes available:");
    Iterator<Class<? extends Attribute>> iter = stream.getAttributeClassesIterator();
    while (iter.hasNext()) {
        Class<? extends Attribute> c = iter.next();
        System.out.println(" - " + c.getCanonicalName());
    }
    System.out.println("");
    // We're now going to iterate through a few tweets and tokenize each in turn.
    for (String tweet : famousTweets) {
        // We're first going to demonstrate the "token-by-token" method of consuming tweets.
        System.out.println("Processing: " + tweet);
        // Reset the token stream to process new input.
        stream.reset(tweet);
        // Now we're going to consume tokens from the stream.
        int tokenCnt = 0;
        while (stream.incrementToken()) {
            // CharSequenceTermAttribute holds the actual token text. This is preferred over
            // TermAttribute because it avoids creating new String objects.
            CharSequenceTermAttribute termAttribute = stream.getAttribute(CharSequenceTermAttribute.class);
            // TokenTypeAttribute holds, as you'd expect, the type of the token.
            TokenTypeAttribute typeAttribute = stream.getAttribute(TokenTypeAttribute.class);
            System.out.println(String.format("token %2d (%3d, %3d) type: %12s, token: '%s'", tokenCnt, termAttribute.getOffset(), termAttribute.getLength() - termAttribute.getOffset(), typeAttribute.getType().name, termAttribute.getTermCharSequence()));
            tokenCnt++;
        }
        System.out.println("");
        // We're now going to demonstrate the TokenizedCharSequence API.
        // This should produce exactly the same result as above.
        tokenCnt = 0;
        System.out.println("Processing: " + tweet);
        TokenizedCharSequence tokSeq = tokenizer.tokenize(tweet);
        for (Token tok : tokSeq.getTokens()) {
            System.out.println(String.format("token %2d (%3d, %3d) type: %12s, token: '%s'", tokenCnt, tok.getOffset(), tok.getOffset() + tok.getLength(), tok.getType().name, tok.getTerm()));
            tokenCnt++;
        }
        System.out.println("");
    }
}

Example 2 with Token

use of com.twitter.common.text.token.TokenizedCharSequence.Token in project commons by twitter.

the class TokenizedCharSequenceTest method testTokenizeToken.

@Test
public void testTokenizeToken() {
    String text = "abCDef";
    TokenizedCharSequence tokenized = new TokenizedCharSequence.Builder(text).addToken(0, 2, TokenType.TOKEN).addToken(2, 2, TokenType.TOKEN).addToken(4, 2, TokenType.TOKEN).build();
    List<Token> tokens = tokenized.getTokens();
    assertEquals(3, tokens.size());
    // tokenize "CD" into "C" and "D"
    Token tokenC = tokens.get(1).tokenize(0, 1);
    Token tokenD = tokens.get(1).tokenize(1, 1);
    assertEquals("C", tokenC.toString());
    assertEquals(2, tokenC.getOffset());
    assertEquals(1, tokenC.getLength());
    assertEquals("D", tokenD.toString());
    assertEquals(3, tokenD.getOffset());
    assertEquals(1, tokenD.getLength());
}