Examples with PosTaggedToken com.joliciel.talismane.posTagger.PosTaggedToken used on opensource projects

Example 1 with PosTaggedToken

use of com.joliciel.talismane.posTagger.PosTaggedToken in project talismane by joliciel-informatique.

the class TransitionBasedParser method parseSentence.

@Override
public List<ParseConfiguration> parseSentence(List<PosTagSequence> input) throws TalismaneException, IOException {
    List<PosTagSequence> posTagSequences = null;
    if (this.propagatePosTaggerBeam) {
        posTagSequences = input;
    } else {
        posTagSequences = new ArrayList<>(1);
        posTagSequences.add(input.get(0));
    }
    long startTime = System.currentTimeMillis();
    int maxAnalysisTimeMilliseconds = maxAnalysisTimePerSentence * 1000;
    int minFreeMemoryBytes = minFreeMemory * KILOBYTE;
    TokenSequence tokenSequence = posTagSequences.get(0).getTokenSequence();
    TreeMap<Integer, PriorityQueue<ParseConfiguration>> heaps = new TreeMap<>();
    PriorityQueue<ParseConfiguration> heap0 = new PriorityQueue<>();
    for (PosTagSequence posTagSequence : posTagSequences) {
        // add an initial ParseConfiguration for each postag sequence
        ParseConfiguration initialConfiguration = new ParseConfiguration(posTagSequence);
        initialConfiguration.setScoringStrategy(decisionMaker.getDefaultScoringStrategy());
        heap0.add(initialConfiguration);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding initial posTagSequence: " + posTagSequence);
        }
    }
    heaps.put(0, heap0);
    PriorityQueue<ParseConfiguration> backupHeap = null;
    PriorityQueue<ParseConfiguration> finalHeap = null;
    PriorityQueue<ParseConfiguration> terminalHeap = new PriorityQueue<>();
    while (heaps.size() > 0) {
        Entry<Integer, PriorityQueue<ParseConfiguration>> heapEntry = heaps.pollFirstEntry();
        PriorityQueue<ParseConfiguration> currentHeap = heapEntry.getValue();
        int currentHeapIndex = heapEntry.getKey();
        if (LOG.isTraceEnabled()) {
            LOG.trace("##### Polling next heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size());
        }
        boolean finished = false;
        // systematically set the final heap here, just in case we exit
        // "naturally" with no more heaps
        finalHeap = heapEntry.getValue();
        backupHeap = new PriorityQueue<>();
        // we jump out when either (a) all tokens have been attached or
        // (b) we go over the max alloted time
        ParseConfiguration topConf = currentHeap.peek();
        if (topConf.isTerminal()) {
            LOG.trace("Exiting with terminal heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size());
            finished = true;
        }
        if (earlyStop && terminalHeap.size() >= beamWidth) {
            LOG.debug("Early stop activated and terminal heap contains " + beamWidth + " entries. Exiting.");
            finalHeap = terminalHeap;
            finished = true;
        }
        long analysisTime = System.currentTimeMillis() - startTime;
        if (maxAnalysisTimePerSentence > 0 && analysisTime > maxAnalysisTimeMilliseconds) {
            LOG.info("Parse tree analysis took too long for sentence: " + tokenSequence.getSentence().getText());
            LOG.info("Breaking out after " + maxAnalysisTimePerSentence + " seconds.");
            finished = true;
        }
        if (minFreeMemory > 0) {
            long freeMemory = Runtime.getRuntime().freeMemory();
            if (freeMemory < minFreeMemoryBytes) {
                LOG.info("Not enough memory left to parse sentence: " + tokenSequence.getSentence().getText());
                LOG.info("Min free memory (bytes):" + minFreeMemoryBytes);
                LOG.info("Current free memory (bytes): " + freeMemory);
                finished = true;
            }
        }
        if (finished) {
            break;
        }
        // limit the breadth to K
        int maxSequences = currentHeap.size() > this.beamWidth ? this.beamWidth : currentHeap.size();
        int j = 0;
        while (currentHeap.size() > 0) {
            ParseConfiguration history = currentHeap.poll();
            if (LOG.isTraceEnabled()) {
                LOG.trace("### Next configuration on heap " + heapEntry.getKey() + ":");
                LOG.trace(history.toString());
                LOG.trace("Score: " + df.format(history.getScore()));
                LOG.trace(history.getPosTagSequence().toString());
            }
            List<Decision> decisions = new ArrayList<>();
            // test the positive rules on the current configuration
            boolean ruleApplied = false;
            if (parserPositiveRules != null) {
                for (ParserRule rule : parserPositiveRules) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Checking rule: " + rule.toString());
                    }
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env);
                    if (ruleResult != null && ruleResult.getOutcome()) {
                        Decision positiveRuleDecision = new Decision(rule.getTransition().getCode());
                        decisions.add(positiveRuleDecision);
                        positiveRuleDecision.addAuthority(rule.getCondition().getName());
                        ruleApplied = true;
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Rule applies. Setting transition to: " + rule.getTransition().getCode());
                        }
                        break;
                    }
                }
            }
            if (!ruleApplied) {
                // test the features on the current configuration
                List<FeatureResult<?>> parseFeatureResults = new ArrayList<>();
                for (ParseConfigurationFeature<?> feature : this.parseFeatures) {
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<?> featureResult = feature.check(history, env);
                    if (featureResult != null)
                        parseFeatureResults.add(featureResult);
                }
                if (LOG_FEATURES.isTraceEnabled()) {
                    SortedSet<String> featureResultSet = parseFeatureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<>()));
                    for (String featureResultString : featureResultSet) {
                        LOG_FEATURES.trace(featureResultString);
                    }
                }
                // evaluate the feature results using the decision maker
                decisions = this.decisionMaker.decide(parseFeatureResults);
                for (ClassificationObserver observer : this.observers) {
                    observer.onAnalyse(history, parseFeatureResults, decisions);
                }
                List<Decision> decisionShortList = new ArrayList<>(decisions.size());
                for (Decision decision : decisions) {
                    if (decision.getProbability() > MIN_PROB_TO_STORE)
                        decisionShortList.add(decision);
                }
                decisions = decisionShortList;
                // apply the negative rules
                Set<String> eliminatedTransitions = new HashSet<>();
                if (parserNegativeRules != null) {
                    for (ParserRule rule : parserNegativeRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking negative rule: " + rule.toString());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            for (Transition transition : rule.getTransitions()) {
                                eliminatedTransitions.add(transition.getCode());
                                if (LOG.isTraceEnabled())
                                    LOG.trace("Rule applies. Eliminating transition: " + transition.getCode());
                            }
                        }
                    }
                    if (eliminatedTransitions.size() > 0) {
                        decisionShortList = new ArrayList<>();
                        for (Decision decision : decisions) {
                            if (!eliminatedTransitions.contains(decision.getOutcome())) {
                                decisionShortList.add(decision);
                            } else {
                                LOG.trace("Eliminating decision: " + decision.toString());
                            }
                        }
                        if (decisionShortList.size() > 0) {
                            decisions = decisionShortList;
                        } else {
                            LOG.debug("All decisions eliminated! Restoring original decisions.");
                        }
                    }
                }
            }
            // has a positive rule been applied?
            boolean transitionApplied = false;
            TransitionSystem transitionSystem = TalismaneSession.get(sessionId).getTransitionSystem();
            // type, we should be able to stop
            for (Decision decision : decisions) {
                Transition transition = transitionSystem.getTransitionForCode(decision.getOutcome());
                if (LOG.isTraceEnabled())
                    LOG.trace("Outcome: " + transition.getCode() + ", " + decision.getProbability());
                if (transition.checkPreconditions(history)) {
                    transitionApplied = true;
                    ParseConfiguration configuration = new ParseConfiguration(history);
                    if (decision.isStatistical())
                        configuration.addDecision(decision);
                    transition.apply(configuration);
                    int nextHeapIndex = parseComparisonStrategy.getComparisonIndex(configuration) * 1000;
                    if (configuration.isTerminal()) {
                        nextHeapIndex = Integer.MAX_VALUE;
                    } else {
                        while (nextHeapIndex <= currentHeapIndex) nextHeapIndex++;
                    }
                    PriorityQueue<ParseConfiguration> nextHeap = heaps.get(nextHeapIndex);
                    if (nextHeap == null) {
                        if (configuration.isTerminal())
                            nextHeap = terminalHeap;
                        else
                            nextHeap = new PriorityQueue<>();
                        heaps.put(nextHeapIndex, nextHeap);
                        if (LOG.isTraceEnabled())
                            LOG.trace("Created heap with index: " + nextHeapIndex);
                    }
                    nextHeap.add(configuration);
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Added configuration with score " + configuration.getScore() + " to heap: " + nextHeapIndex + ", total size: " + nextHeap.size());
                    }
                    configuration.clearMemory();
                } else {
                    if (LOG.isTraceEnabled())
                        LOG.trace("Cannot apply transition: doesn't meet pre-conditions");
                    // just in case the we run out of both heaps and
                    // analyses, we build this backup heap
                    backupHeap.add(history);
                }
            // does transition meet pre-conditions?
            }
            if (transitionApplied) {
                j++;
            } else {
                LOG.trace("No transitions could be applied: not counting this history as part of the beam");
            }
            // beam width test
            if (j == maxSequences)
                break;
        }
    // next history
    }
    // next atomic index
    // return the best sequences on the heap
    List<ParseConfiguration> bestConfigurations = new ArrayList<>();
    int i = 0;
    if (finalHeap.isEmpty())
        finalHeap = backupHeap;
    while (!finalHeap.isEmpty()) {
        bestConfigurations.add(finalHeap.poll());
        i++;
        if (i >= this.getBeamWidth())
            break;
    }
    if (LOG.isDebugEnabled()) {
        for (ParseConfiguration finalConfiguration : bestConfigurations) {
            LOG.debug(df.format(finalConfiguration.getScore()) + ": " + finalConfiguration.toString());
            LOG.debug("Pos tag sequence: " + finalConfiguration.getPosTagSequence());
            LOG.debug("Transitions: " + finalConfiguration.getTransitions());
            LOG.debug("Decisions: " + finalConfiguration.getDecisions());
            if (LOG.isTraceEnabled()) {
                StringBuilder sb = new StringBuilder();
                for (Decision decision : finalConfiguration.getDecisions()) {
                    sb.append(" * ");
                    sb.append(df.format(decision.getProbability()));
                }
                sb.append(" root ");
                sb.append(finalConfiguration.getTransitions().size());
                LOG.trace(sb.toString());
                sb = new StringBuilder();
                sb.append(" * PosTag sequence score ");
                sb.append(df.format(finalConfiguration.getPosTagSequence().getScore()));
                sb.append(" = ");
                for (PosTaggedToken posTaggedToken : finalConfiguration.getPosTagSequence()) {
                    sb.append(" * ");
                    sb.append(df.format(posTaggedToken.getDecision().getProbability()));
                }
                sb.append(" root ");
                sb.append(finalConfiguration.getPosTagSequence().size());
                LOG.trace(sb.toString());
                sb = new StringBuilder();
                sb.append(" * Token sequence score = ");
                sb.append(df.format(finalConfiguration.getPosTagSequence().getTokenSequence().getScore()));
                LOG.trace(sb.toString());
            }
        }
    }
    return bestConfigurations;
}

Example 2 with PosTaggedToken

use of com.joliciel.talismane.posTagger.PosTaggedToken in project talismane by joliciel-informatique.

the class ParseConfiguration method getTransition.

/**
 * Get the transition which generated the dependency arc provided.
 */
public Transition getTransition(DependencyArc arc) {
    PosTaggedToken dependent = arc.getDependent();
    Transition transition = this.dependentTransitionMap[dependent.getIndex()];
    return transition;
}

Example 3 with PosTaggedToken

use of com.joliciel.talismane.posTagger.PosTaggedToken in project talismane by joliciel-informatique.

the class ParseConfiguration method addManualNonProjectiveDependency.

/**
 * Add the given dependency to the current configuration's non-projective
 * dependency set. This should only be used when reading a previously
 * annotated corpus, to indicate the projective and non-projective governor
 * for a given token. If the transition system is capable of producing its own
 * non-projective dependencies there should be no need to distinguish between
 * projective and non-projective.
 *
 * @throws CircularDependencyException
 *           if this would create a circular dependency
 */
public DependencyArc addManualNonProjectiveDependency(PosTaggedToken head, PosTaggedToken dependent, String label) throws CircularDependencyException {
    if (!this.hasNonProjDependencies) {
        this.hasNonProjDependencies = true;
    }
    DependencyArc arc = new DependencyArc(head, dependent, label);
    PosTaggedToken ancestor = arc.getHead();
    while (ancestor != null) {
        if (ancestor.equals(arc.getDependent())) {
            throw new CircularDependencyException(this, arc.getHead(), arc.getDependent());
        }
        ancestor = this.getHead(ancestor, false);
    }
    this.dependenciesNonProj.add(arc);
    this.governingDependencyNonProjMap[arc.getDependent().getIndex()] = arc;
    this.governorNonProjMap[arc.getDependent().getIndex()] = arc.getHead().getIndex();
    this.labelNonProjMap[arc.getDependent().getIndex()] = arc.getLabel();
    return arc;
}

Example 4 with PosTaggedToken

use of com.joliciel.talismane.posTagger.PosTaggedToken in project talismane by joliciel-informatique.

the class ParseTreeNode method getGapCount.

/**
 * Non-projectivity: gives the number of "gaps" in the node's yield. Each gap
 * is defined as a series of one or more tokens which are not members of the
 * node's yield, but which are contained between two tokens which are members
 * of the node's yield.
 */
public int getGapCount() {
    Set<PosTaggedToken> yield = this.getYield();
    int currentIndex = -1;
    int gapCount = 0;
    for (PosTaggedToken token : yield) {
        if (currentIndex < 0) {
        // do nothing: first token
        } else if (token.getIndex() == currentIndex + 1) {
        // do nothing: not a gap
        } else {
            gapCount++;
        }
        currentIndex = token.getIndex();
    }
    return gapCount;
}

Example 5 with PosTaggedToken

use of com.joliciel.talismane.posTagger.PosTaggedToken in project talismane by joliciel-informatique.

the class ParseTreeNode method getMyNonProjectiveEdges.

private void getMyNonProjectiveEdges(List<DependencyArc> nonProjectiveEdges) {
    Set<PosTaggedToken> yield = this.getYield();
    for (ParseTreeNode child : this.getChildren()) {
        int i = this.getIndex() < child.getIndex() ? this.getIndex() : child.getIndex();
        int j = this.getIndex() > child.getIndex() ? this.getIndex() : child.getIndex();
        boolean projective = true;
        for (int k = i + 1; k < j; k++) {
            PosTaggedToken other = this.parseTree.getPosTaggedTokens().get(k);
            if (!yield.contains(other)) {
                projective = false;
                break;
            }
        }
        if (!projective) {
            DependencyArc arc = this.parseTree.getGoverningArc(child.getPosTaggedToken());
            if (arc != null)
                nonProjectiveEdges.add(arc);
        }
    }
}