Examples with TokenSequence com.joliciel.talismane.tokeniser.TokenSequence used on opensource projects

Example 1 with TokenSequence

use of com.joliciel.talismane.tokeniser.TokenSequence in project talismane by joliciel-informatique.

the class TransitionBasedParser method parseSentence.

@Override
public List<ParseConfiguration> parseSentence(List<PosTagSequence> input) throws TalismaneException, IOException {
    List<PosTagSequence> posTagSequences = null;
    if (this.propagatePosTaggerBeam) {
        posTagSequences = input;
    } else {
        posTagSequences = new ArrayList<>(1);
        posTagSequences.add(input.get(0));
    }
    long startTime = System.currentTimeMillis();
    int maxAnalysisTimeMilliseconds = maxAnalysisTimePerSentence * 1000;
    int minFreeMemoryBytes = minFreeMemory * KILOBYTE;
    TokenSequence tokenSequence = posTagSequences.get(0).getTokenSequence();
    TreeMap<Integer, PriorityQueue<ParseConfiguration>> heaps = new TreeMap<>();
    PriorityQueue<ParseConfiguration> heap0 = new PriorityQueue<>();
    for (PosTagSequence posTagSequence : posTagSequences) {
        // add an initial ParseConfiguration for each postag sequence
        ParseConfiguration initialConfiguration = new ParseConfiguration(posTagSequence);
        initialConfiguration.setScoringStrategy(decisionMaker.getDefaultScoringStrategy());
        heap0.add(initialConfiguration);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding initial posTagSequence: " + posTagSequence);
        }
    }
    heaps.put(0, heap0);
    PriorityQueue<ParseConfiguration> backupHeap = null;
    PriorityQueue<ParseConfiguration> finalHeap = null;
    PriorityQueue<ParseConfiguration> terminalHeap = new PriorityQueue<>();
    while (heaps.size() > 0) {
        Entry<Integer, PriorityQueue<ParseConfiguration>> heapEntry = heaps.pollFirstEntry();
        PriorityQueue<ParseConfiguration> currentHeap = heapEntry.getValue();
        int currentHeapIndex = heapEntry.getKey();
        if (LOG.isTraceEnabled()) {
            LOG.trace("##### Polling next heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size());
        }
        boolean finished = false;
        // systematically set the final heap here, just in case we exit
        // "naturally" with no more heaps
        finalHeap = heapEntry.getValue();
        backupHeap = new PriorityQueue<>();
        // we jump out when either (a) all tokens have been attached or
        // (b) we go over the max alloted time
        ParseConfiguration topConf = currentHeap.peek();
        if (topConf.isTerminal()) {
            LOG.trace("Exiting with terminal heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size());
            finished = true;
        }
        if (earlyStop && terminalHeap.size() >= beamWidth) {
            LOG.debug("Early stop activated and terminal heap contains " + beamWidth + " entries. Exiting.");
            finalHeap = terminalHeap;
            finished = true;
        }
        long analysisTime = System.currentTimeMillis() - startTime;
        if (maxAnalysisTimePerSentence > 0 && analysisTime > maxAnalysisTimeMilliseconds) {
            LOG.info("Parse tree analysis took too long for sentence: " + tokenSequence.getSentence().getText());
            LOG.info("Breaking out after " + maxAnalysisTimePerSentence + " seconds.");
            finished = true;
        }
        if (minFreeMemory > 0) {
            long freeMemory = Runtime.getRuntime().freeMemory();
            if (freeMemory < minFreeMemoryBytes) {
                LOG.info("Not enough memory left to parse sentence: " + tokenSequence.getSentence().getText());
                LOG.info("Min free memory (bytes):" + minFreeMemoryBytes);
                LOG.info("Current free memory (bytes): " + freeMemory);
                finished = true;
            }
        }
        if (finished) {
            break;
        }
        // limit the breadth to K
        int maxSequences = currentHeap.size() > this.beamWidth ? this.beamWidth : currentHeap.size();
        int j = 0;
        while (currentHeap.size() > 0) {
            ParseConfiguration history = currentHeap.poll();
            if (LOG.isTraceEnabled()) {
                LOG.trace("### Next configuration on heap " + heapEntry.getKey() + ":");
                LOG.trace(history.toString());
                LOG.trace("Score: " + df.format(history.getScore()));
                LOG.trace(history.getPosTagSequence().toString());
            }
            List<Decision> decisions = new ArrayList<>();
            // test the positive rules on the current configuration
            boolean ruleApplied = false;
            if (parserPositiveRules != null) {
                for (ParserRule rule : parserPositiveRules) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Checking rule: " + rule.toString());
                    }
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env);
                    if (ruleResult != null && ruleResult.getOutcome()) {
                        Decision positiveRuleDecision = new Decision(rule.getTransition().getCode());
                        decisions.add(positiveRuleDecision);
                        positiveRuleDecision.addAuthority(rule.getCondition().getName());
                        ruleApplied = true;
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Rule applies. Setting transition to: " + rule.getTransition().getCode());
                        }
                        break;
                    }
                }
            }
            if (!ruleApplied) {
                // test the features on the current configuration
                List<FeatureResult<?>> parseFeatureResults = new ArrayList<>();
                for (ParseConfigurationFeature<?> feature : this.parseFeatures) {
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<?> featureResult = feature.check(history, env);
                    if (featureResult != null)
                        parseFeatureResults.add(featureResult);
                }
                if (LOG_FEATURES.isTraceEnabled()) {
                    SortedSet<String> featureResultSet = parseFeatureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<>()));
                    for (String featureResultString : featureResultSet) {
                        LOG_FEATURES.trace(featureResultString);
                    }
                }
                // evaluate the feature results using the decision maker
                decisions = this.decisionMaker.decide(parseFeatureResults);
                for (ClassificationObserver observer : this.observers) {
                    observer.onAnalyse(history, parseFeatureResults, decisions);
                }
                List<Decision> decisionShortList = new ArrayList<>(decisions.size());
                for (Decision decision : decisions) {
                    if (decision.getProbability() > MIN_PROB_TO_STORE)
                        decisionShortList.add(decision);
                }
                decisions = decisionShortList;
                // apply the negative rules
                Set<String> eliminatedTransitions = new HashSet<>();
                if (parserNegativeRules != null) {
                    for (ParserRule rule : parserNegativeRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking negative rule: " + rule.toString());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            for (Transition transition : rule.getTransitions()) {
                                eliminatedTransitions.add(transition.getCode());
                                if (LOG.isTraceEnabled())
                                    LOG.trace("Rule applies. Eliminating transition: " + transition.getCode());
                            }
                        }
                    }
                    if (eliminatedTransitions.size() > 0) {
                        decisionShortList = new ArrayList<>();
                        for (Decision decision : decisions) {
                            if (!eliminatedTransitions.contains(decision.getOutcome())) {
                                decisionShortList.add(decision);
                            } else {
                                LOG.trace("Eliminating decision: " + decision.toString());
                            }
                        }
                        if (decisionShortList.size() > 0) {
                            decisions = decisionShortList;
                        } else {
                            LOG.debug("All decisions eliminated! Restoring original decisions.");
                        }
                    }
                }
            }
            // has a positive rule been applied?
            boolean transitionApplied = false;
            TransitionSystem transitionSystem = TalismaneSession.get(sessionId).getTransitionSystem();
            // type, we should be able to stop
            for (Decision decision : decisions) {
                Transition transition = transitionSystem.getTransitionForCode(decision.getOutcome());
                if (LOG.isTraceEnabled())
                    LOG.trace("Outcome: " + transition.getCode() + ", " + decision.getProbability());
                if (transition.checkPreconditions(history)) {
                    transitionApplied = true;
                    ParseConfiguration configuration = new ParseConfiguration(history);
                    if (decision.isStatistical())
                        configuration.addDecision(decision);
                    transition.apply(configuration);
                    int nextHeapIndex = parseComparisonStrategy.getComparisonIndex(configuration) * 1000;
                    if (configuration.isTerminal()) {
                        nextHeapIndex = Integer.MAX_VALUE;
                    } else {
                        while (nextHeapIndex <= currentHeapIndex) nextHeapIndex++;
                    }
                    PriorityQueue<ParseConfiguration> nextHeap = heaps.get(nextHeapIndex);
                    if (nextHeap == null) {
                        if (configuration.isTerminal())
                            nextHeap = terminalHeap;
                        else
                            nextHeap = new PriorityQueue<>();
                        heaps.put(nextHeapIndex, nextHeap);
                        if (LOG.isTraceEnabled())
                            LOG.trace("Created heap with index: " + nextHeapIndex);
                    }
                    nextHeap.add(configuration);
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Added configuration with score " + configuration.getScore() + " to heap: " + nextHeapIndex + ", total size: " + nextHeap.size());
                    }
                    configuration.clearMemory();
                } else {
                    if (LOG.isTraceEnabled())
                        LOG.trace("Cannot apply transition: doesn't meet pre-conditions");
                    // just in case the we run out of both heaps and
                    // analyses, we build this backup heap
                    backupHeap.add(history);
                }
            // does transition meet pre-conditions?
            }
            if (transitionApplied) {
                j++;
            } else {
                LOG.trace("No transitions could be applied: not counting this history as part of the beam");
            }
            // beam width test
            if (j == maxSequences)
                break;
        }
    // next history
    }
    // next atomic index
    // return the best sequences on the heap
    List<ParseConfiguration> bestConfigurations = new ArrayList<>();
    int i = 0;
    if (finalHeap.isEmpty())
        finalHeap = backupHeap;
    while (!finalHeap.isEmpty()) {
        bestConfigurations.add(finalHeap.poll());
        i++;
        if (i >= this.getBeamWidth())
            break;
    }
    if (LOG.isDebugEnabled()) {
        for (ParseConfiguration finalConfiguration : bestConfigurations) {
            LOG.debug(df.format(finalConfiguration.getScore()) + ": " + finalConfiguration.toString());
            LOG.debug("Pos tag sequence: " + finalConfiguration.getPosTagSequence());
            LOG.debug("Transitions: " + finalConfiguration.getTransitions());
            LOG.debug("Decisions: " + finalConfiguration.getDecisions());
            if (LOG.isTraceEnabled()) {
                StringBuilder sb = new StringBuilder();
                for (Decision decision : finalConfiguration.getDecisions()) {
                    sb.append(" * ");
                    sb.append(df.format(decision.getProbability()));
                }
                sb.append(" root ");
                sb.append(finalConfiguration.getTransitions().size());
                LOG.trace(sb.toString());
                sb = new StringBuilder();
                sb.append(" * PosTag sequence score ");
                sb.append(df.format(finalConfiguration.getPosTagSequence().getScore()));
                sb.append(" = ");
                for (PosTaggedToken posTaggedToken : finalConfiguration.getPosTagSequence()) {
                    sb.append(" * ");
                    sb.append(df.format(posTaggedToken.getDecision().getProbability()));
                }
                sb.append(" root ");
                sb.append(finalConfiguration.getPosTagSequence().size());
                LOG.trace(sb.toString());
                sb = new StringBuilder();
                sb.append(" * Token sequence score = ");
                sb.append(df.format(finalConfiguration.getPosTagSequence().getTokenSequence().getScore()));
                LOG.trace(sb.toString());
            }
        }
    }
    return bestConfigurations;
}

Example 2 with TokenSequence

use of com.joliciel.talismane.tokeniser.TokenSequence in project talismane by joliciel-informatique.

the class SentenceDetector method detectSentences.

/**
 * Detect sentences within an annotated text. Sentences are added in the form
 * of an Annotation around a {@link SentenceBoundary}, with the start position
 * (relative to the start of the annotated text) at the start of the sentence
 * and the end position immediately after the end of the sentence. <br>
 * <br>
 * Sentence boundaries will not be detected within any annotation of type
 * {@link RawTextNoSentenceBreakMarker}, nor will they be detected before or
 * after the {@link AnnotatedText#getAnalysisStart()} and
 * {@link AnnotatedText#getAnalysisEnd()} respectively. <br>
 * <br>
 * If the text contained existing {@link SentenceBoundary} annotations before
 * analysis start, the first sentence will begin where the last existing
 * annotation ended. Otherwise, the first boundary will begin at position 0.
 * <br>
 * <br>
 * If the text's analysis end is equal to the text length, it is assumed that
 * the text end is a sentence boundary. In this case, an additional sentence
 * is added starting at the final detected boundary and ending at text end.
 *
 * @param text
 *          the annotated text in which we need to detect sentences.
 * @return in addition to the annotations added, we return a List of integers
 *         marking the end position of each sentence boundary.
 */
public List<Integer> detectSentences(AnnotatedText text, String... labels) throws TalismaneException {
    LOG.debug("detectSentences");
    List<Annotation<RawTextNoSentenceBreakMarker>> noSentenceBreakMarkers = text.getAnnotations(RawTextNoSentenceBreakMarker.class);
    Matcher matcher = possibleBoundaryPattern.matcher(text.getText());
    List<Integer> possibleBoundaries = new ArrayList<>();
    while (matcher.find()) {
        if (matcher.start() >= text.getAnalysisStart() && matcher.start() < text.getAnalysisEnd()) {
            boolean noSentences = false;
            int position = matcher.start();
            for (Annotation<RawTextNoSentenceBreakMarker> noSentenceBreakMarker : noSentenceBreakMarkers) {
                if (noSentenceBreakMarker.getStart() <= position && position < noSentenceBreakMarker.getEnd()) {
                    noSentences = true;
                    break;
                }
            }
            if (!noSentences)
                possibleBoundaries.add(position);
        }
    }
    // collect all deterministic sentence boundaries
    List<Annotation<RawTextSentenceBreakMarker>> sentenceBreakMarkers = text.getAnnotations(RawTextSentenceBreakMarker.class);
    Set<Integer> guessedBoundaries = new TreeSet<>(sentenceBreakMarkers.stream().filter(f -> f.getEnd() >= text.getAnalysisStart()).map(f -> f.getEnd()).collect(Collectors.toList()));
    // Share one token sequence for all possible boundaries, to avoid tokenising
    // multiple times
    Sentence sentence = new Sentence(text.getText(), sessionId);
    TokenSequence tokenSequence = new TokenSequence(sentence, sessionId);
    List<PossibleSentenceBoundary> boundaries = new ArrayList<>();
    for (int possibleBoundary : possibleBoundaries) {
        PossibleSentenceBoundary boundary = new PossibleSentenceBoundary(tokenSequence, possibleBoundary);
        if (LOG.isTraceEnabled()) {
            LOG.trace("Testing boundary: " + boundary);
            LOG.trace(" at position: " + possibleBoundary);
        }
        List<FeatureResult<?>> featureResults = new ArrayList<>();
        for (SentenceDetectorFeature<?> feature : features) {
            RuntimeEnvironment env = new RuntimeEnvironment();
            FeatureResult<?> featureResult = feature.check(boundary, env);
            if (featureResult != null)
                featureResults.add(featureResult);
        }
        if (LOG.isTraceEnabled()) {
            SortedSet<String> featureResultSet = featureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>()));
            for (String featureResultString : featureResultSet) {
                LOG.trace(featureResultString);
            }
        }
        List<Decision> decisions = this.decisionMaker.decide(featureResults);
        if (LOG.isTraceEnabled()) {
            for (Decision decision : decisions) {
                LOG.trace(decision.getOutcome() + ": " + decision.getProbability());
            }
        }
        if (decisions.get(0).getOutcome().equals(SentenceDetectorOutcome.IS_BOUNDARY.name())) {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Adding boundary: " + possibleBoundary + 1);
            }
            guessedBoundaries.add(possibleBoundary + 1);
            boundaries.add(boundary);
        }
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("context: " + text.getText().toString().replace('\n', '¶').replace('\r', '¶'));
        for (PossibleSentenceBoundary boundary : boundaries) LOG.trace("boundary: " + boundary.toString());
    }
    if (LOG.isDebugEnabled())
        LOG.debug("guessedBoundaries : " + guessedBoundaries.toString());
    List<Annotation<SentenceBoundary>> newBoundaries = new ArrayList<>();
    int lastBoundary = 0;
    List<Annotation<SentenceBoundary>> existingBoundaries = text.getAnnotations(SentenceBoundary.class);
    if (existingBoundaries.size() > 0) {
        lastBoundary = existingBoundaries.get(existingBoundaries.size() - 1).getEnd();
    }
    // advance boundary start until a non space character is encountered
    while (lastBoundary < text.getAnalysisEnd() && Character.isWhitespace(text.getText().charAt(lastBoundary))) {
        lastBoundary++;
    }
    for (int guessedBoundary : guessedBoundaries) {
        if (guessedBoundary > lastBoundary) {
            Annotation<SentenceBoundary> sentenceBoundary = new Annotation<>(lastBoundary, guessedBoundary, new SentenceBoundary(), labels);
            newBoundaries.add(sentenceBoundary);
            if (LOG.isTraceEnabled()) {
                LOG.trace("Added boundary: " + sentenceBoundary);
            }
            lastBoundary = guessedBoundary;
        }
    }
    if (text.getAnalysisEnd() == text.getText().length()) {
        if (text.getAnalysisEnd() > lastBoundary) {
            Annotation<SentenceBoundary> sentenceBoundary = new Annotation<>(lastBoundary, text.getAnalysisEnd(), new SentenceBoundary(), labels);
            newBoundaries.add(sentenceBoundary);
            if (LOG.isTraceEnabled()) {
                LOG.trace("Added final boundary: " + sentenceBoundary);
            }
        }
    }
    text.addAnnotations(newBoundaries);
    return new ArrayList<>(guessedBoundaries);
}

Example 3 with TokenSequence

use of com.joliciel.talismane.tokeniser.TokenSequence in project talismane by joliciel-informatique.

the class ParserEvaluator method evaluate.

/**
 * @throws TalismaneException
 *           if an attempt is made to evaluate with a tokeniser but no
 *           pos-tagger
 * @throws IOException
 */
public void evaluate() throws TalismaneException, IOException {
    while (corpusReader.hasNextSentence()) {
        ParseConfiguration realConfiguration = corpusReader.nextConfiguration();
        List<PosTagSequence> posTagSequences = null;
        List<TokenSequence> tokenSequences = null;
        if (tokeniser != null) {
            if (posTagger == null)
                throw new TalismaneException("Cannot evaluate with tokeniser but no pos-tagger");
            Sentence sentence = realConfiguration.getPosTagSequence().getTokenSequence().getSentence();
            // annotate the sentence for pre token filters
            for (SentenceAnnotator annotator : TalismaneSession.get(sessionId).getSentenceAnnotators()) {
                annotator.annotate(sentence);
                if (LOG.isTraceEnabled()) {
                    LOG.trace("TokenFilter: " + annotator);
                    LOG.trace("annotations: " + sentence.getAnnotations());
                }
            }
            tokenSequences = tokeniser.tokenise(sentence);
        } else {
            tokenSequences = new ArrayList<TokenSequence>();
            PosTagSequence posTagSequence = realConfiguration.getPosTagSequence().clonePosTagSequence();
            posTagSequence.removeRoot();
            tokenSequences.add(posTagSequence.getTokenSequence());
        }
        if (posTagger != null) {
            if (posTagger instanceof NonDeterministicPosTagger) {
                NonDeterministicPosTagger nonDeterministicPosTagger = (NonDeterministicPosTagger) posTagger;
                posTagSequences = nonDeterministicPosTagger.tagSentence(tokenSequences);
            } else {
                posTagSequences = new ArrayList<PosTagSequence>();
                PosTagSequence posTagSequence = null;
                posTagSequence = posTagger.tagSentence(tokenSequences.get(0));
                posTagSequences.add(posTagSequence);
            }
        } else {
            PosTagSequence posTagSequence = realConfiguration.getPosTagSequence();
            posTagSequences = new ArrayList<PosTagSequence>();
            posTagSequences.add(posTagSequence);
        }
        for (ParseEvaluationObserver observer : this.observers) {
            observer.onParseStart(realConfiguration, posTagSequences);
        }
        List<ParseConfiguration> guessedConfigurations = null;
        if (parser instanceof NonDeterministicParser) {
            NonDeterministicParser nonDeterministicParser = (NonDeterministicParser) parser;
            guessedConfigurations = nonDeterministicParser.parseSentence(posTagSequences);
        } else {
            ParseConfiguration bestGuess = parser.parseSentence(posTagSequences.get(0));
            guessedConfigurations = new ArrayList<ParseConfiguration>();
            guessedConfigurations.add(bestGuess);
        }
        for (ParseEvaluationObserver observer : this.observers) {
            observer.onParseEnd(realConfiguration, guessedConfigurations);
        }
    }
    for (ParseEvaluationObserver observer : this.observers) {
        observer.onEvaluationComplete();
    }
}

Example 4 with TokenSequence

use of com.joliciel.talismane.tokeniser.TokenSequence in project talismane by joliciel-informatique.

the class PosTaggerEvaluator method evaluate.

/**
 * Evaluate a given pos tagger.
 *
 * @throws TalismaneException
 * @throws IOException
 */
public void evaluate() throws TalismaneException, IOException {
    while (corpusReader.hasNextSentence()) {
        PosTagSequence realPosTagSequence = corpusReader.nextPosTagSequence();
        List<TokenSequence> tokenSequences = null;
        List<PosTagSequence> guessedSequences = null;
        TokenSequence tokenSequence = realPosTagSequence.getTokenSequence();
        PosTagSequence guessedSequence = null;
        if (this.tokeniser != null) {
            Sentence sentence = tokenSequence.getSentence();
            tokenSequences = tokeniser.tokenise(sentence);
            tokenSequence = tokenSequences.get(0);
        } else {
            tokenSequences = new ArrayList<TokenSequence>();
            tokenSequences.add(tokenSequence);
        }
        if (posTagger instanceof NonDeterministicPosTagger) {
            NonDeterministicPosTagger nonDeterministicPosTagger = (NonDeterministicPosTagger) posTagger;
            guessedSequences = nonDeterministicPosTagger.tagSentence(tokenSequences);
            guessedSequence = guessedSequences.get(0);
        } else {
            guessedSequence = posTagger.tagSentence(tokenSequence);
        }
        if (LOG.isDebugEnabled()) {
            StringBuilder stringBuilder = new StringBuilder();
            for (PosTaggedToken posTaggedToken : guessedSequence) {
                Set<String> lemmas = new TreeSet<String>();
                stringBuilder.append(posTaggedToken.getToken().getOriginalText());
                stringBuilder.append("[" + posTaggedToken.getTag());
                List<LexicalEntry> entries = posTaggedToken.getLexicalEntries();
                boolean dropCurrentWord = false;
                if (entries.size() > 1)
                    dropCurrentWord = true;
                for (LexicalEntry entry : posTaggedToken.getLexicalEntries()) {
                    if (!lemmas.contains(entry.getLemma())) {
                        if (dropCurrentWord && posTaggedToken.getToken().getText().equals(entry.getLemma())) {
                            dropCurrentWord = false;
                            continue;
                        }
                        stringBuilder.append("|" + entry.getLemma());
                        // stringBuilder.append("/" + entry.getCategory());
                        stringBuilder.append("/" + entry.getMorphology());
                        lemmas.add(entry.getLemma());
                    }
                }
                stringBuilder.append("] ");
            }
            LOG.debug(stringBuilder.toString());
        }
        for (PosTagEvaluationObserver observer : this.observers) {
            observer.onNextPosTagSequence(realPosTagSequence, guessedSequences);
        }
    }
    for (PosTagEvaluationObserver observer : this.observers) {
        observer.onEvaluationComplete();
    }
}

Example 5 with TokenSequence

use of com.joliciel.talismane.tokeniser.TokenSequence in project talismane by joliciel-informatique.

the class ForwardStatisticalPosTagger method tagSentence.

@Override
public List<PosTagSequence> tagSentence(List<TokenSequence> input) throws TalismaneException, IOException {
    List<TokenSequence> tokenSequences = null;
    if (this.propagateTokeniserBeam) {
        tokenSequences = input;
    } else {
        tokenSequences = new ArrayList<>(1);
        tokenSequences.add(input.get(0));
    }
    int sentenceLength = tokenSequences.get(0).getSentence().getText().length();
    TreeMap<Double, PriorityQueue<PosTagSequence>> heaps = new TreeMap<Double, PriorityQueue<PosTagSequence>>();
    PriorityQueue<PosTagSequence> heap0 = new PriorityQueue<PosTagSequence>();
    for (TokenSequence tokenSequence : tokenSequences) {
        // add an empty PosTagSequence for each token sequence
        PosTagSequence emptySequence = new PosTagSequence(tokenSequence);
        emptySequence.setScoringStrategy(decisionMaker.getDefaultScoringStrategy());
        heap0.add(emptySequence);
    }
    heaps.put(0.0, heap0);
    PriorityQueue<PosTagSequence> finalHeap = null;
    while (heaps.size() > 0) {
        Entry<Double, PriorityQueue<PosTagSequence>> heapEntry = heaps.pollFirstEntry();
        if (LOG.isTraceEnabled()) {
            LOG.trace("heap key: " + heapEntry.getKey() + ", sentence length: " + sentenceLength);
        }
        if (heapEntry.getKey() == sentenceLength) {
            finalHeap = heapEntry.getValue();
            break;
        }
        PriorityQueue<PosTagSequence> previousHeap = heapEntry.getValue();
        // limit the breadth to K
        int maxSequences = previousHeap.size() > this.beamWidth ? this.beamWidth : previousHeap.size();
        for (int j = 0; j < maxSequences; j++) {
            PosTagSequence history = previousHeap.poll();
            Token token = history.getNextToken();
            if (LOG.isTraceEnabled()) {
                LOG.trace("#### Next history ( " + heapEntry.getKey() + "): " + history.toString());
                LOG.trace("Prob: " + df.format(history.getScore()));
                LOG.trace("Token: " + token.getText());
                StringBuilder sb = new StringBuilder();
                for (Token oneToken : history.getTokenSequence().listWithWhiteSpace()) {
                    if (oneToken.equals(token))
                        sb.append("[" + oneToken + "]");
                    else
                        sb.append(oneToken);
                }
                LOG.trace(sb.toString());
            }
            PosTaggerContext context = new PosTaggerContextImpl(token, history);
            List<Decision> decisions = new ArrayList<Decision>();
            boolean ruleApplied = false;
            // assigned?
            if (token.getAttributes().containsKey(PosTagger.POS_TAG_ATTRIBUTE)) {
                StringAttribute posTagCodeAttribute = (StringAttribute) token.getAttributes().get(PosTagger.POS_TAG_ATTRIBUTE);
                String posTagCode = posTagCodeAttribute.getValue();
                Decision positiveRuleDecision = new Decision(posTagCode);
                decisions.add(positiveRuleDecision);
                positiveRuleDecision.addAuthority("tokenAttribute");
                ruleApplied = true;
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Token has attribute \"" + PosTagger.POS_TAG_ATTRIBUTE + "\". Setting posTag to: " + posTagCode);
                }
            }
            // test the positive rules on the current token
            if (!ruleApplied) {
                if (posTaggerPositiveRules != null) {
                    for (PosTaggerRule rule : posTaggerPositiveRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking rule: " + rule.getCondition().getName());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(context, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            Decision positiveRuleDecision = new Decision(rule.getTag().getCode());
                            decisions.add(positiveRuleDecision);
                            positiveRuleDecision.addAuthority(rule.getCondition().getName());
                            ruleApplied = true;
                            if (LOG.isTraceEnabled()) {
                                LOG.trace("Rule applies. Setting posTag to: " + rule.getTag().getCode());
                            }
                            break;
                        }
                    }
                }
            }
            if (!ruleApplied) {
                // test the features on the current token
                List<FeatureResult<?>> featureResults = new ArrayList<FeatureResult<?>>();
                for (PosTaggerFeature<?> posTaggerFeature : posTaggerFeatures) {
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<?> featureResult = posTaggerFeature.check(context, env);
                    if (featureResult != null)
                        featureResults.add(featureResult);
                }
                if (LOG.isTraceEnabled()) {
                    SortedSet<String> featureResultSet = featureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>()));
                    for (String featureResultString : featureResultSet) {
                        LOG.trace(featureResultString);
                    }
                }
                // evaluate the feature results using the maxent model
                decisions = this.decisionMaker.decide(featureResults);
                for (ClassificationObserver observer : this.observers) {
                    observer.onAnalyse(token, featureResults, decisions);
                }
                // apply the negative rules
                Set<String> eliminatedPosTags = new TreeSet<String>();
                if (posTaggerNegativeRules != null) {
                    for (PosTaggerRule rule : posTaggerNegativeRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking negative rule: " + rule.getCondition().getName());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(context, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            eliminatedPosTags.add(rule.getTag().getCode());
                            if (LOG.isTraceEnabled()) {
                                LOG.trace("Rule applies. Eliminating posTag: " + rule.getTag().getCode());
                            }
                        }
                    }
                    if (eliminatedPosTags.size() > 0) {
                        List<Decision> decisionShortList = new ArrayList<Decision>();
                        for (Decision decision : decisions) {
                            if (!eliminatedPosTags.contains(decision.getOutcome())) {
                                decisionShortList.add(decision);
                            } else {
                                LOG.trace("Eliminating decision: " + decision.toString());
                            }
                        }
                        if (decisionShortList.size() > 0) {
                            decisions = decisionShortList;
                        } else {
                            LOG.debug("All decisions eliminated! Restoring original decisions.");
                        }
                    }
                }
                // is this a known word in the lexicon?
                if (LOG.isTraceEnabled()) {
                    String posTags = "";
                    for (PosTag onePosTag : token.getPossiblePosTags()) {
                        posTags += onePosTag.getCode() + ",";
                    }
                    LOG.trace("Token: " + token.getText() + ". PosTags: " + posTags);
                }
                List<Decision> decisionShortList = new ArrayList<Decision>();
                for (Decision decision : decisions) {
                    if (decision.getProbability() >= MIN_PROB_TO_STORE) {
                        decisionShortList.add(decision);
                    }
                }
                if (decisionShortList.size() > 0) {
                    decisions = decisionShortList;
                }
            }
            // outcome provided by MaxEnt
            for (Decision decision : decisions) {
                if (LOG.isTraceEnabled())
                    LOG.trace("Outcome: " + decision.getOutcome() + ", " + decision.getProbability());
                PosTaggedToken posTaggedToken = new PosTaggedToken(token, decision, this.sessionId);
                PosTagSequence sequence = new PosTagSequence(history);
                sequence.addPosTaggedToken(posTaggedToken);
                if (decision.isStatistical())
                    sequence.addDecision(decision);
                double heapIndex = token.getEndIndex();
                // it from regular ones
                if (token.getStartIndex() == token.getEndIndex())
                    heapIndex += 0.5;
                // if it's the last token, make sure we end
                if (token.getIndex() == sequence.getTokenSequence().size() - 1)
                    heapIndex = sentenceLength;
                if (LOG.isTraceEnabled())
                    LOG.trace("Heap index: " + heapIndex);
                PriorityQueue<PosTagSequence> heap = heaps.get(heapIndex);
                if (heap == null) {
                    heap = new PriorityQueue<PosTagSequence>();
                    heaps.put(heapIndex, heap);
                }
                heap.add(sequence);
            }
        // next outcome for this token
        }
    // next history
    }
    // next atomic index
    // return the best sequence on the heap
    List<PosTagSequence> sequences = new ArrayList<PosTagSequence>();
    int i = 0;
    while (!finalHeap.isEmpty()) {
        // clone the pos tag sequences to ensure they don't share any underlying
        // data (e.g. token sequences)
        sequences.add(finalHeap.poll().clonePosTagSequence());
        i++;
        if (i >= this.getBeamWidth())
            break;
    }
    // apply post-processing filters
    if (LOG.isDebugEnabled()) {
        LOG.debug("####Final postag sequences:");
        int j = 1;
        for (PosTagSequence sequence : sequences) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sequence " + (j++) + ", score=" + df.format(sequence.getScore()));
                LOG.debug("Sequence: " + sequence);
            }
        }
    }
    return sequences;
}