Examples with OffsetAttribute org.apache.lucene.analysis.tokenattributes.OffsetAttribute used on opensource projects

Example 41 with OffsetAttribute

use of org.apache.lucene.analysis.tokenattributes.OffsetAttribute in project lucene-solr by apache.

the class SynonymTokenizer method getTS2.

protected TokenStream getTS2() {
    // String s = "Hi-Speed10 foo";
    return new TokenStream() {

        Iterator<Token> iter;

        List<Token> lst;

        private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

        private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);

        private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);

        {
            lst = new ArrayList<>();
            Token t;
            t = createToken("hi", 0, 2);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("hispeed", 0, 8);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("speed", 3, 8);
            t.setPositionIncrement(0);
            lst.add(t);
            t = createToken("10", 8, 10);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("foo", 11, 14);
            t.setPositionIncrement(1);
            lst.add(t);
            iter = lst.iterator();
        }

        @Override
        public boolean incrementToken() {
            if (iter.hasNext()) {
                Token token = iter.next();
                clearAttributes();
                termAtt.setEmpty().append(token);
                posIncrAtt.setPositionIncrement(token.getPositionIncrement());
                offsetAtt.setOffset(token.startOffset(), token.endOffset());
                return true;
            }
            return false;
        }

        @Override
        public void reset() throws IOException {
            super.reset();
            iter = lst.iterator();
        }
    };
}

Example 42 with OffsetAttribute

use of org.apache.lucene.analysis.tokenattributes.OffsetAttribute in project lucene-solr by apache.

the class SynonymTokenizer method getTS2a.

// same token-stream as above, but the bigger token comes first this time
protected TokenStream getTS2a() {
    // String s = "Hi-Speed10 foo";
    return new TokenStream() {

        Iterator<Token> iter;

        List<Token> lst;

        private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

        private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);

        private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);

        {
            lst = new ArrayList<>();
            Token t;
            t = createToken("hispeed", 0, 8);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("hi", 0, 2);
            t.setPositionIncrement(0);
            lst.add(t);
            t = createToken("speed", 3, 8);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("10", 8, 10);
            t.setPositionIncrement(1);
            lst.add(t);
            t = createToken("foo", 11, 14);
            t.setPositionIncrement(1);
            lst.add(t);
            iter = lst.iterator();
        }

        @Override
        public boolean incrementToken() {
            if (iter.hasNext()) {
                Token token = iter.next();
                clearAttributes();
                termAtt.setEmpty().append(token);
                posIncrAtt.setPositionIncrement(token.getPositionIncrement());
                offsetAtt.setOffset(token.startOffset(), token.endOffset());
                return true;
            }
            return false;
        }

        @Override
        public void reset() throws IOException {
            super.reset();
            iter = lst.iterator();
        }
    };
}

Example 43 with OffsetAttribute

use of org.apache.lucene.analysis.tokenattributes.OffsetAttribute in project lucene-solr by apache.

the class NGramTokenizerTest method testNGrams.

static void testNGrams(int minGram, int maxGram, String s, final String nonTokenChars, boolean edgesOnly) throws IOException {
    // convert the string to code points
    final int[] codePoints = toCodePoints(s);
    final int[] offsets = new int[codePoints.length + 1];
    for (int i = 0; i < codePoints.length; ++i) {
        offsets[i + 1] = offsets[i] + Character.charCount(codePoints[i]);
    }
    final Tokenizer grams = new NGramTokenizer(minGram, maxGram, edgesOnly) {

        @Override
        protected boolean isTokenChar(int chr) {
            return nonTokenChars.indexOf(chr) < 0;
        }
    };
    grams.setReader(new StringReader(s));
    final CharTermAttribute termAtt = grams.addAttribute(CharTermAttribute.class);
    final PositionIncrementAttribute posIncAtt = grams.addAttribute(PositionIncrementAttribute.class);
    final PositionLengthAttribute posLenAtt = grams.addAttribute(PositionLengthAttribute.class);
    final OffsetAttribute offsetAtt = grams.addAttribute(OffsetAttribute.class);
    grams.reset();
    for (int start = 0; start < codePoints.length; ++start) {
        nextGram: for (int end = start + minGram; end <= start + maxGram && end <= codePoints.length; ++end) {
            if (edgesOnly && start > 0 && isTokenChar(nonTokenChars, codePoints[start - 1])) {
                // not on an edge
                continue nextGram;
            }
            for (int j = start; j < end; ++j) {
                if (!isTokenChar(nonTokenChars, codePoints[j])) {
                    continue nextGram;
                }
            }
            assertTrue(grams.incrementToken());
            assertArrayEquals(Arrays.copyOfRange(codePoints, start, end), toCodePoints(termAtt));
            assertEquals(1, posIncAtt.getPositionIncrement());
            assertEquals(1, posLenAtt.getPositionLength());
            assertEquals(offsets[start], offsetAtt.startOffset());
            assertEquals(offsets[end], offsetAtt.endOffset());
        }
    }
    assertFalse(grams.incrementToken());
    grams.end();
    assertEquals(s.length(), offsetAtt.startOffset());
    assertEquals(s.length(), offsetAtt.endOffset());
}

Example 44 with OffsetAttribute

use of org.apache.lucene.analysis.tokenattributes.OffsetAttribute in project lucene-solr by apache.

the class TestSimplePatternTokenizer method testEndOffset.

public void testEndOffset() throws Exception {
    Tokenizer t = new SimplePatternTokenizer("a+");
    CharTermAttribute termAtt = t.getAttribute(CharTermAttribute.class);
    OffsetAttribute offsetAtt = t.getAttribute(OffsetAttribute.class);
    t.setReader(new StringReader("aaabbb"));
    t.reset();
    assertTrue(t.incrementToken());
    assertEquals("aaa", termAtt.toString());
    assertFalse(t.incrementToken());
    t.end();
    assertEquals(6, offsetAtt.endOffset());
}

Example 45 with OffsetAttribute

use of org.apache.lucene.analysis.tokenattributes.OffsetAttribute in project lucene-solr by apache.

the class TestRemoveDuplicatesTokenFilter method testDups.

public void testDups(final String expected, final Token... tokens) throws Exception {
    final Iterator<Token> toks = Arrays.asList(tokens).iterator();
    final TokenStream ts = new RemoveDuplicatesTokenFilter((new TokenStream() {

        CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

        OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);

        PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);

        @Override
        public boolean incrementToken() {
            if (toks.hasNext()) {
                clearAttributes();
                Token tok = toks.next();
                termAtt.setEmpty().append(tok);
                offsetAtt.setOffset(tok.startOffset(), tok.endOffset());
                posIncAtt.setPositionIncrement(tok.getPositionIncrement());
                return true;
            } else {
                return false;
            }
        }
    }));
    assertTokenStreamContents(ts, expected.split("\\s"));
}