Examples with AnnotationIteratorPair opennlp.uima.util.AnnotationIteratorPair used on opensource projects

Example 1 with AnnotationIteratorPair

use of opennlp.uima.util.AnnotationIteratorPair in project deeplearning4j by deeplearning4j.

the class PoStagger method process.

/**
     * Performs pos-tagging on the given tcas object.
     */
@Override
public synchronized void process(CAS tcas) {
    final AnnotationComboIterator comboIterator = new AnnotationComboIterator(tcas, this.sentenceType, this.tokenType);
    for (AnnotationIteratorPair annotationIteratorPair : comboIterator) {
        final List<AnnotationFS> sentenceTokenAnnotationList = new LinkedList<>();
        final List<String> sentenceTokenList = new LinkedList<>();
        for (AnnotationFS tokenAnnotation : annotationIteratorPair.getSubIterator()) {
            sentenceTokenAnnotationList.add(tokenAnnotation);
            sentenceTokenList.add(tokenAnnotation.getCoveredText());
        }
        final List<String> posTags = this.posTagger.tag(sentenceTokenList);
        double[] posProbabilities = null;
        if (this.probabilityFeature != null) {
            posProbabilities = this.posTagger.probs();
        }
        final Iterator<String> posTagIterator = posTags.iterator();
        final Iterator<AnnotationFS> sentenceTokenIterator = sentenceTokenAnnotationList.iterator();
        int index = 0;
        while (posTagIterator.hasNext() && sentenceTokenIterator.hasNext()) {
            final String posTag = posTagIterator.next();
            final AnnotationFS tokenAnnotation = sentenceTokenIterator.next();
            tokenAnnotation.setStringValue(this.posFeature, posTag);
            if (posProbabilities != null) {
                tokenAnnotation.setDoubleValue(this.posFeature, posProbabilities[index]);
            }
            index++;
        }
        // log tokens with pos
        if (this.logger.isLoggable(Level.FINER)) {
            final StringBuilder sentenceWithPos = new StringBuilder();
            sentenceWithPos.append("\"");
            for (final Iterator<AnnotationFS> it = sentenceTokenAnnotationList.iterator(); it.hasNext(); ) {
                final AnnotationFS token = it.next();
                sentenceWithPos.append(token.getCoveredText());
                sentenceWithPos.append('\\');
                sentenceWithPos.append(token.getStringValue(this.posFeature));
                sentenceWithPos.append(' ');
            }
            // delete last whitespace
            if (// not 0 because it contains already the " char
            sentenceWithPos.length() > 1)
                sentenceWithPos.setLength(sentenceWithPos.length() - 1);
            sentenceWithPos.append("\"");
            this.logger.log(Level.FINER, sentenceWithPos.toString());
        }
    }
}