Examples with PackedTokenAttributeImpl org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl used on opensource projects

Example 1 with PackedTokenAttributeImpl

use of org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl in project SearchServices by Alfresco.

the class MLTokenDuplicator method next.

private PackedTokenAttributeImpl next() throws IOException {
    PackedTokenAttributeImpl t = null;
    if (it == null) {
        it = buildIterator();
    }
    if (it == null) {
        return null;
    }
    if (it.hasNext()) {
        t = it.next();
        return t;
    } else {
        it = null;
        t = this.next();
        return t;
    }
}

Example 2 with PackedTokenAttributeImpl

use of org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl in project SearchServices by Alfresco.

the class MLTokenDuplicator method incrementToken.

@Override
public final boolean incrementToken() throws IOException {
    clearAttributes();
    PackedTokenAttributeImpl next = next();
    if (next == null) {
        return false;
    }
    termAtt.copyBuffer(next.buffer(), 0, next.length());
    offsetAtt.setOffset(next.startOffset(), next.endOffset());
    typeAtt.setType(next.type());
    posIncAtt.setPositionIncrement(next.getPositionIncrement());
    return true;
}

Example 3 with PackedTokenAttributeImpl

use of org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl in project SearchServices by Alfresco.

the class MLTokenDuplicator method buildIterator.

private Iterator<PackedTokenAttributeImpl> buildIterator() throws IOException {
    // TODO: use incrementToken() somehow?
    if (!done && source.incrementToken()) {
        CharTermAttribute cta = source.getAttribute(CharTermAttribute.class);
        OffsetAttribute offsetAtt = source.getAttribute(OffsetAttribute.class);
        TypeAttribute typeAtt = null;
        if (source.hasAttribute(TypeAttribute.class)) {
            typeAtt = source.getAttribute(TypeAttribute.class);
        }
        PositionIncrementAttribute posIncAtt = null;
        if (source.hasAttribute(PositionIncrementAttribute.class)) {
            posIncAtt = source.getAttribute(PositionIncrementAttribute.class);
        }
        PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
        token.setEmpty().append(new String(cta.buffer()), 0, cta.length());
        token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
        if (typeAtt != null) {
            token.setType(typeAtt.type());
        }
        if (posIncAtt != null) {
            token.setPositionIncrement(posIncAtt.getPositionIncrement());
        }
        return buildIterator(token);
    } else {
        done = true;
        return buildIterator(null);
    }
}

Example 4 with PackedTokenAttributeImpl

use of org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl in project SearchServices by Alfresco.

the class MLTokenDuplicator method buildIterator.

public Iterator<PackedTokenAttributeImpl> buildIterator(PackedTokenAttributeImpl token) {
    if (token == null) {
        return null;
    }
    ArrayList<PackedTokenAttributeImpl> tokens = new ArrayList<PackedTokenAttributeImpl>(prefixes.size());
    for (String prefix : prefixes) {
        PackedTokenAttributeImpl newToken = new PackedTokenAttributeImpl();
        newToken.setEmpty().append(prefix + termText(token));
        newToken.setOffset(token.startOffset(), token.endOffset());
        newToken.setType(token.type());
        if (tokens.size() == 0) {
            newToken.setPositionIncrement(token.getPositionIncrement());
        } else {
            newToken.setPositionIncrement(0);
        }
        tokens.add(newToken);
    }
    return tokens.iterator();
}

Example 5 with PackedTokenAttributeImpl

use of org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl in project SearchServices by Alfresco.

the class Solr4QueryParserTest method getTokenAttribute.

private PackedTokenAttributeImpl getTokenAttribute(String text, int startOffset, int endOffset) {
    PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
    token.setEmpty().append(text);
    token.setOffset(startOffset, endOffset);
    return token;
}