Examples with FeatureResult com.joliciel.talismane.machineLearning.features.FeatureResult used on opensource projects

Example 6 with FeatureResult

use of com.joliciel.talismane.machineLearning.features.FeatureResult in project talismane by joliciel-informatique.

the class PosTagFeatureTester method onNextPosTagSequence.

@Override
public void onNextPosTagSequence(PosTagSequence posTagSequence) throws TalismaneException {
    PosTagSequence currentHistory = new PosTagSequence(posTagSequence.getTokenSequence());
    for (PosTaggedToken posTaggedToken : posTagSequence) {
        if (testWords.contains(posTaggedToken.getToken().getAnalyisText().toLowerCase())) {
            StringBuilder sb = new StringBuilder();
            boolean foundToken = false;
            for (PosTaggedToken taggedToken : posTagSequence) {
                if (taggedToken.equals(posTaggedToken)) {
                    sb.append(" [" + taggedToken.getToken().getOriginalText().replace(' ', '_') + "/" + taggedToken.getTag().toString() + "]");
                    foundToken = true;
                } else if (foundToken) {
                    sb.append(" " + taggedToken.getToken().getOriginalText().replace(' ', '_'));
                } else {
                    sb.append(" " + taggedToken.getToken().getOriginalText().replace(' ', '_') + "/" + taggedToken.getTag().toString());
                }
            }
            LOG.debug(sb.toString());
            String classification = posTaggedToken.getTag().getCode();
            PosTaggerContext context = new PosTaggerContextImpl(posTaggedToken.getToken(), currentHistory);
            List<FeatureResult<?>> posTagFeatureResults = new ArrayList<FeatureResult<?>>();
            for (PosTaggerFeature<?> posTaggerFeature : posTaggerFeatures) {
                RuntimeEnvironment env = new RuntimeEnvironment();
                FeatureResult<?> featureResult = posTaggerFeature.check(context, env);
                if (featureResult != null)
                    posTagFeatureResults.add(featureResult);
            }
            if (LOG.isTraceEnabled()) {
                LOG.trace("Token: " + posTaggedToken.getToken().getAnalyisText());
                for (FeatureResult<?> result : posTagFeatureResults) {
                    LOG.trace(result.toString());
                }
            }
            for (FeatureResult<?> featureResult : posTagFeatureResults) {
                Map<String, List<String>> classificationMap = featureResultMap.get(featureResult.toString());
                if (classificationMap == null) {
                    classificationMap = new TreeMap<String, List<String>>();
                    featureResultMap.put(featureResult.toString(), classificationMap);
                }
                List<String> sentences = classificationMap.get(classification);
                if (sentences == null) {
                    sentences = new ArrayList<String>();
                    classificationMap.put(classification, sentences);
                }
                sentences.add(sb.toString());
            }
        }
        currentHistory.addPosTaggedToken(posTaggedToken);
    }
}

Example 7 with FeatureResult

use of com.joliciel.talismane.machineLearning.features.FeatureResult in project talismane by joliciel-informatique.

the class ParseFeatureTester method onNextParseConfiguration.

@Override
public void onNextParseConfiguration(ParseConfiguration parseConfiguration) throws TalismaneException {
    ParseConfiguration currentConfiguration = new ParseConfiguration(parseConfiguration.getPosTagSequence());
    for (Transition transition : parseConfiguration.getTransitions()) {
        StringBuilder sb = new StringBuilder();
        for (PosTaggedToken taggedToken : currentConfiguration.getPosTagSequence()) {
            if (taggedToken.equals(currentConfiguration.getStack().getFirst())) {
                sb.append(" #[" + taggedToken.getToken().getOriginalText().replace(' ', '_') + "/" + taggedToken.getTag().toString() + "]#");
            } else if (taggedToken.equals(currentConfiguration.getBuffer().getFirst())) {
                sb.append(" #[" + taggedToken.getToken().getOriginalText().replace(' ', '_') + "/" + taggedToken.getTag().toString() + "]#");
            } else {
                sb.append(" " + taggedToken.getToken().getOriginalText().replace(' ', '_') + "/" + taggedToken.getTag().toString());
            }
        }
        sb.append(" ## Line: " + parseConfiguration.getSentence().getStartLineNumber());
        if (LOG.isTraceEnabled())
            LOG.trace(sb.toString());
        List<FeatureResult<?>> parseFeatureResults = new ArrayList<FeatureResult<?>>();
        for (ParseConfigurationFeature<?> parseFeature : parseFeatures) {
            RuntimeEnvironment env = new RuntimeEnvironment();
            FeatureResult<?> featureResult = parseFeature.check(currentConfiguration, env);
            if (featureResult != null) {
                parseFeatureResults.add(featureResult);
                if (LOG.isTraceEnabled()) {
                    LOG.trace(featureResult.toString());
                }
            }
        }
        String classification = transition.getCode();
        for (FeatureResult<?> featureResult : parseFeatureResults) {
            Map<String, List<String>> classificationMap = featureResultMap.get(featureResult.toString());
            if (classificationMap == null) {
                classificationMap = new TreeMap<String, List<String>>();
                featureResultMap.put(featureResult.toString(), classificationMap);
            }
            List<String> sentences = classificationMap.get(classification);
            if (sentences == null) {
                sentences = new ArrayList<String>();
                classificationMap.put(classification, sentences);
            }
            sentences.add(sb.toString());
        }
        // apply the transition and up the index
        currentConfiguration = new ParseConfiguration(currentConfiguration);
        transition.apply(currentConfiguration);
    }
}

Example 8 with FeatureResult

use of com.joliciel.talismane.machineLearning.features.FeatureResult in project talismane by joliciel-informatique.

the class ForwardStatisticalPosTagger method tagSentence.

@Override
public List<PosTagSequence> tagSentence(List<TokenSequence> input) throws TalismaneException, IOException {
    List<TokenSequence> tokenSequences = null;
    if (this.propagateTokeniserBeam) {
        tokenSequences = input;
    } else {
        tokenSequences = new ArrayList<>(1);
        tokenSequences.add(input.get(0));
    }
    int sentenceLength = tokenSequences.get(0).getSentence().getText().length();
    TreeMap<Double, PriorityQueue<PosTagSequence>> heaps = new TreeMap<Double, PriorityQueue<PosTagSequence>>();
    PriorityQueue<PosTagSequence> heap0 = new PriorityQueue<PosTagSequence>();
    for (TokenSequence tokenSequence : tokenSequences) {
        // add an empty PosTagSequence for each token sequence
        PosTagSequence emptySequence = new PosTagSequence(tokenSequence);
        emptySequence.setScoringStrategy(decisionMaker.getDefaultScoringStrategy());
        heap0.add(emptySequence);
    }
    heaps.put(0.0, heap0);
    PriorityQueue<PosTagSequence> finalHeap = null;
    while (heaps.size() > 0) {
        Entry<Double, PriorityQueue<PosTagSequence>> heapEntry = heaps.pollFirstEntry();
        if (LOG.isTraceEnabled()) {
            LOG.trace("heap key: " + heapEntry.getKey() + ", sentence length: " + sentenceLength);
        }
        if (heapEntry.getKey() == sentenceLength) {
            finalHeap = heapEntry.getValue();
            break;
        }
        PriorityQueue<PosTagSequence> previousHeap = heapEntry.getValue();
        // limit the breadth to K
        int maxSequences = previousHeap.size() > this.beamWidth ? this.beamWidth : previousHeap.size();
        for (int j = 0; j < maxSequences; j++) {
            PosTagSequence history = previousHeap.poll();
            Token token = history.getNextToken();
            if (LOG.isTraceEnabled()) {
                LOG.trace("#### Next history ( " + heapEntry.getKey() + "): " + history.toString());
                LOG.trace("Prob: " + df.format(history.getScore()));
                LOG.trace("Token: " + token.getText());
                StringBuilder sb = new StringBuilder();
                for (Token oneToken : history.getTokenSequence().listWithWhiteSpace()) {
                    if (oneToken.equals(token))
                        sb.append("[" + oneToken + "]");
                    else
                        sb.append(oneToken);
                }
                LOG.trace(sb.toString());
            }
            PosTaggerContext context = new PosTaggerContextImpl(token, history);
            List<Decision> decisions = new ArrayList<Decision>();
            boolean ruleApplied = false;
            // assigned?
            if (token.getAttributes().containsKey(PosTagger.POS_TAG_ATTRIBUTE)) {
                StringAttribute posTagCodeAttribute = (StringAttribute) token.getAttributes().get(PosTagger.POS_TAG_ATTRIBUTE);
                String posTagCode = posTagCodeAttribute.getValue();
                Decision positiveRuleDecision = new Decision(posTagCode);
                decisions.add(positiveRuleDecision);
                positiveRuleDecision.addAuthority("tokenAttribute");
                ruleApplied = true;
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Token has attribute \"" + PosTagger.POS_TAG_ATTRIBUTE + "\". Setting posTag to: " + posTagCode);
                }
            }
            // test the positive rules on the current token
            if (!ruleApplied) {
                if (posTaggerPositiveRules != null) {
                    for (PosTaggerRule rule : posTaggerPositiveRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking rule: " + rule.getCondition().getName());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(context, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            Decision positiveRuleDecision = new Decision(rule.getTag().getCode());
                            decisions.add(positiveRuleDecision);
                            positiveRuleDecision.addAuthority(rule.getCondition().getName());
                            ruleApplied = true;
                            if (LOG.isTraceEnabled()) {
                                LOG.trace("Rule applies. Setting posTag to: " + rule.getTag().getCode());
                            }
                            break;
                        }
                    }
                }
            }
            if (!ruleApplied) {
                // test the features on the current token
                List<FeatureResult<?>> featureResults = new ArrayList<FeatureResult<?>>();
                for (PosTaggerFeature<?> posTaggerFeature : posTaggerFeatures) {
                    RuntimeEnvironment env = new RuntimeEnvironment();
                    FeatureResult<?> featureResult = posTaggerFeature.check(context, env);
                    if (featureResult != null)
                        featureResults.add(featureResult);
                }
                if (LOG.isTraceEnabled()) {
                    SortedSet<String> featureResultSet = featureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>()));
                    for (String featureResultString : featureResultSet) {
                        LOG.trace(featureResultString);
                    }
                }
                // evaluate the feature results using the maxent model
                decisions = this.decisionMaker.decide(featureResults);
                for (ClassificationObserver observer : this.observers) {
                    observer.onAnalyse(token, featureResults, decisions);
                }
                // apply the negative rules
                Set<String> eliminatedPosTags = new TreeSet<String>();
                if (posTaggerNegativeRules != null) {
                    for (PosTaggerRule rule : posTaggerNegativeRules) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Checking negative rule: " + rule.getCondition().getName());
                        }
                        RuntimeEnvironment env = new RuntimeEnvironment();
                        FeatureResult<Boolean> ruleResult = rule.getCondition().check(context, env);
                        if (ruleResult != null && ruleResult.getOutcome()) {
                            eliminatedPosTags.add(rule.getTag().getCode());
                            if (LOG.isTraceEnabled()) {
                                LOG.trace("Rule applies. Eliminating posTag: " + rule.getTag().getCode());
                            }
                        }
                    }
                    if (eliminatedPosTags.size() > 0) {
                        List<Decision> decisionShortList = new ArrayList<Decision>();
                        for (Decision decision : decisions) {
                            if (!eliminatedPosTags.contains(decision.getOutcome())) {
                                decisionShortList.add(decision);
                            } else {
                                LOG.trace("Eliminating decision: " + decision.toString());
                            }
                        }
                        if (decisionShortList.size() > 0) {
                            decisions = decisionShortList;
                        } else {
                            LOG.debug("All decisions eliminated! Restoring original decisions.");
                        }
                    }
                }
                // is this a known word in the lexicon?
                if (LOG.isTraceEnabled()) {
                    String posTags = "";
                    for (PosTag onePosTag : token.getPossiblePosTags()) {
                        posTags += onePosTag.getCode() + ",";
                    }
                    LOG.trace("Token: " + token.getText() + ". PosTags: " + posTags);
                }
                List<Decision> decisionShortList = new ArrayList<Decision>();
                for (Decision decision : decisions) {
                    if (decision.getProbability() >= MIN_PROB_TO_STORE) {
                        decisionShortList.add(decision);
                    }
                }
                if (decisionShortList.size() > 0) {
                    decisions = decisionShortList;
                }
            }
            // outcome provided by MaxEnt
            for (Decision decision : decisions) {
                if (LOG.isTraceEnabled())
                    LOG.trace("Outcome: " + decision.getOutcome() + ", " + decision.getProbability());
                PosTaggedToken posTaggedToken = new PosTaggedToken(token, decision, this.sessionId);
                PosTagSequence sequence = new PosTagSequence(history);
                sequence.addPosTaggedToken(posTaggedToken);
                if (decision.isStatistical())
                    sequence.addDecision(decision);
                double heapIndex = token.getEndIndex();
                // it from regular ones
                if (token.getStartIndex() == token.getEndIndex())
                    heapIndex += 0.5;
                // if it's the last token, make sure we end
                if (token.getIndex() == sequence.getTokenSequence().size() - 1)
                    heapIndex = sentenceLength;
                if (LOG.isTraceEnabled())
                    LOG.trace("Heap index: " + heapIndex);
                PriorityQueue<PosTagSequence> heap = heaps.get(heapIndex);
                if (heap == null) {
                    heap = new PriorityQueue<PosTagSequence>();
                    heaps.put(heapIndex, heap);
                }
                heap.add(sequence);
            }
        // next outcome for this token
        }
    // next history
    }
    // next atomic index
    // return the best sequence on the heap
    List<PosTagSequence> sequences = new ArrayList<PosTagSequence>();
    int i = 0;
    while (!finalHeap.isEmpty()) {
        // clone the pos tag sequences to ensure they don't share any underlying
        // data (e.g. token sequences)
        sequences.add(finalHeap.poll().clonePosTagSequence());
        i++;
        if (i >= this.getBeamWidth())
            break;
    }
    // apply post-processing filters
    if (LOG.isDebugEnabled()) {
        LOG.debug("####Final postag sequences:");
        int j = 1;
        for (PosTagSequence sequence : sequences) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sequence " + (j++) + ", score=" + df.format(sequence.getScore()));
                LOG.debug("Sequence: " + sequence);
            }
        }
    }
    return sequences;
}

Example 9 with FeatureResult

use of com.joliciel.talismane.machineLearning.features.FeatureResult in project talismane by joliciel-informatique.

the class PatternTokeniser method tokeniseInternal.

@Override
protected List<TokenisedAtomicTokenSequence> tokeniseInternal(TokenSequence initialSequence, Sentence sentence) throws TalismaneException, IOException {
    List<TokenisedAtomicTokenSequence> sequences;
    // Assign each separator its default value
    List<TokeniserOutcome> defaultOutcomes = this.tokeniserPatternManager.getDefaultOutcomes(initialSequence);
    List<Decision> defaultDecisions = new ArrayList<Decision>(defaultOutcomes.size());
    for (TokeniserOutcome outcome : defaultOutcomes) {
        Decision tokeniserDecision = new Decision(outcome.name());
        tokeniserDecision.addAuthority("_" + this.getClass().getSimpleName());
        tokeniserDecision.addAuthority("_" + "DefaultDecision");
        defaultDecisions.add(tokeniserDecision);
    }
    // For each test pattern, see if anything in the sentence matches it
    if (this.decisionMaker != null) {
        List<TokenPatternMatchSequence> matchingSequences = new ArrayList<TokenPatternMatchSequence>();
        Map<Token, Set<TokenPatternMatchSequence>> tokenMatchSequenceMap = new HashMap<Token, Set<TokenPatternMatchSequence>>();
        Map<TokenPatternMatchSequence, TokenPatternMatch> primaryMatchMap = new HashMap<TokenPatternMatchSequence, TokenPatternMatch>();
        Set<Token> matchedTokens = new HashSet<Token>();
        for (TokenPattern parsedPattern : this.getTokeniserPatternManager().getParsedTestPatterns()) {
            List<TokenPatternMatchSequence> matchesForThisPattern = parsedPattern.match(initialSequence);
            for (TokenPatternMatchSequence matchSequence : matchesForThisPattern) {
                if (matchSequence.getTokensToCheck().size() > 0) {
                    matchingSequences.add(matchSequence);
                    matchedTokens.addAll(matchSequence.getTokensToCheck());
                    TokenPatternMatch primaryMatch = null;
                    Token token = matchSequence.getTokensToCheck().get(0);
                    Set<TokenPatternMatchSequence> matchSequences = tokenMatchSequenceMap.get(token);
                    if (matchSequences == null) {
                        matchSequences = new TreeSet<TokenPatternMatchSequence>();
                        tokenMatchSequenceMap.put(token, matchSequences);
                    }
                    matchSequences.add(matchSequence);
                    for (TokenPatternMatch patternMatch : matchSequence.getTokenPatternMatches()) {
                        if (patternMatch.getToken().equals(token)) {
                            primaryMatch = patternMatch;
                            break;
                        }
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Found match: " + primaryMatch);
                    }
                    primaryMatchMap.put(matchSequence, primaryMatch);
                }
            }
        }
        // we want to create the n most likely token sequences
        // the sequence has to correspond to a token pattern
        Map<TokenPatternMatchSequence, List<Decision>> matchSequenceDecisionMap = new HashMap<TokenPatternMatchSequence, List<Decision>>();
        for (TokenPatternMatchSequence matchSequence : matchingSequences) {
            TokenPatternMatch match = primaryMatchMap.get(matchSequence);
            LOG.debug("next pattern match: " + match.toString());
            List<FeatureResult<?>> tokenFeatureResults = new ArrayList<FeatureResult<?>>();
            for (TokenPatternMatchFeature<?> feature : features) {
                RuntimeEnvironment env = new RuntimeEnvironment();
                FeatureResult<?> featureResult = feature.check(match, env);
                if (featureResult != null) {
                    tokenFeatureResults.add(featureResult);
                }
            }
            if (LOG.isTraceEnabled()) {
                SortedSet<String> featureResultSet = tokenFeatureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>()));
                for (String featureResultString : featureResultSet) {
                    LOG.trace(featureResultString);
                }
            }
            List<Decision> decisions = this.decisionMaker.decide(tokenFeatureResults);
            for (ClassificationObserver observer : this.observers) observer.onAnalyse(match.getToken(), tokenFeatureResults, decisions);
            for (Decision decision : decisions) {
                decision.addAuthority("_" + this.getClass().getSimpleName());
                decision.addAuthority("_" + "Patterns");
                decision.addAuthority(match.getPattern().getName());
            }
            matchSequenceDecisionMap.put(matchSequence, decisions);
        }
        // initially create a heap with a single, empty sequence
        PriorityQueue<TokenisedAtomicTokenSequence> heap = new PriorityQueue<TokenisedAtomicTokenSequence>();
        TokenisedAtomicTokenSequence emptySequence = new TokenisedAtomicTokenSequence(sentence, 0, this.getSessionId());
        heap.add(emptySequence);
        for (int i = 0; i < initialSequence.listWithWhiteSpace().size(); i++) {
            Token token = initialSequence.listWithWhiteSpace().get(i);
            if (LOG.isTraceEnabled()) {
                LOG.trace("Token : \"" + token.getAnalyisText() + "\"");
            }
            // build a new heap for this iteration
            PriorityQueue<TokenisedAtomicTokenSequence> previousHeap = heap;
            heap = new PriorityQueue<TokenisedAtomicTokenSequence>();
            if (i == 0) {
                // first token is always "separate" from the outside world
                Decision decision = new Decision(TokeniserOutcome.SEPARATE.name());
                decision.addAuthority("_" + this.getClass().getSimpleName());
                decision.addAuthority("_" + "DefaultDecision");
                TaggedToken<TokeniserOutcome> taggedToken = new TaggedToken<>(token, decision, TokeniserOutcome.valueOf(decision.getOutcome()));
                TokenisedAtomicTokenSequence newSequence = new TokenisedAtomicTokenSequence(emptySequence);
                newSequence.add(taggedToken);
                heap.add(newSequence);
                continue;
            }
            // limit the heap breadth to K
            int maxSequences = previousHeap.size() > this.getBeamWidth() ? this.getBeamWidth() : previousHeap.size();
            for (int j = 0; j < maxSequences; j++) {
                TokenisedAtomicTokenSequence history = previousHeap.poll();
                // Find the separating & non-separating decisions
                if (history.size() > i) {
                    // token already added as part of a sequence
                    // introduced by another token
                    heap.add(history);
                } else if (tokenMatchSequenceMap.containsKey(token)) {
                    // token begins one or more match sequences
                    // these are ordered from shortest to longest (via
                    // TreeSet)
                    List<TokenPatternMatchSequence> matchSequences = new ArrayList<TokenPatternMatchSequence>(tokenMatchSequenceMap.get(token));
                    // Since sequences P1..Pn contain each other,
                    // there can be exactly matchSequences.size()
                    // consistent solutions
                    // Assume the default is separate
                    // 0: all separate
                    // 1: join P1, separate rest
                    // 2: join P2, separate rest
                    // ...
                    // n: join Pn
                    // We need to add each of these to the heap
                    // by taking the product of all probabilities
                    // consistent with each solution
                    // The probabities for each solution are (j=join,
                    // s=separate)
                    // All separate: s1 x s2 x ... x sn
                    // P1: j1 x s2 x ... x sn
                    // P2: j1 x j2 x ... x sn
                    // ...
                    // Pn: j1 x j2 x ... x jn
                    // Any solution of the form s1 x j2 would be
                    // inconsistent, and is not considered
                    // If Pi and Pj start and end on the exact same
                    // token, then the solution for both is
                    // Pi: j1 x ... x ji x jj x sj+1 ... x sn
                    // Pj: j1 x ... x ji x jj x sj+1 ... x sn
                    // Note of course that we're never likely to have
                    // more than two Ps here,
                    // but we need a solution for more just to be sure
                    // to be sure
                    TokeniserOutcome defaultOutcome = TokeniserOutcome.valueOf(defaultDecisions.get(token.getIndexWithWhiteSpace()).getOutcome());
                    TokeniserOutcome otherOutcome = null;
                    if (defaultOutcome == TokeniserOutcome.SEPARATE)
                        otherOutcome = TokeniserOutcome.JOIN;
                    else
                        otherOutcome = TokeniserOutcome.SEPARATE;
                    double[] decisionProbs = new double[matchSequences.size() + 1];
                    for (int k = 0; k < decisionProbs.length; k++) decisionProbs[k] = 1;
                    // Note: k0 = default decision (e.g. separate all),
                    // k1=first pattern
                    // p1 = first pattern
                    int p = 1;
                    int prevEndIndex = -1;
                    for (TokenPatternMatchSequence matchSequence : matchSequences) {
                        int endIndex = matchSequence.getTokensToCheck().get(matchSequence.getTokensToCheck().size() - 1).getEndIndex();
                        List<Decision> decisions = matchSequenceDecisionMap.get(matchSequence);
                        for (Decision decision : decisions) {
                            for (int k = 0; k < decisionProbs.length; k++) {
                                if (decision.getOutcome().equals(defaultOutcome.name())) {
                                    // e.g. separate in most cases
                                    if (k < p && endIndex > prevEndIndex)
                                        decisionProbs[k] *= decision.getProbability();
                                    else if (k + 1 < p && endIndex <= prevEndIndex)
                                        decisionProbs[k] *= decision.getProbability();
                                } else {
                                    // e.g. join in most cases
                                    if (k >= p && endIndex > prevEndIndex)
                                        decisionProbs[k] *= decision.getProbability();
                                    else if (k + 1 >= p && endIndex <= prevEndIndex)
                                        decisionProbs[k] *= decision.getProbability();
                                }
                            }
                        // next k
                        }
                        // next decision (only 2 of these)
                        prevEndIndex = endIndex;
                        p++;
                    }
                    // transform to probability distribution
                    double sumProbs = 0;
                    for (int k = 0; k < decisionProbs.length; k++) sumProbs += decisionProbs[k];
                    if (sumProbs > 0)
                        for (int k = 0; k < decisionProbs.length; k++) decisionProbs[k] /= sumProbs;
                    // Apply default decision
                    // Since this is the default decision for all tokens
                    // in the sequence, we don't add the other tokens
                    // for now,
                    // so as to allow them
                    // to get examined one at a time, just in case one
                    // of them starts its own separate sequence
                    Decision defaultDecision = new Decision(defaultOutcome.name(), decisionProbs[0]);
                    defaultDecision.addAuthority("_" + this.getClass().getSimpleName());
                    defaultDecision.addAuthority("_" + "Patterns");
                    for (TokenPatternMatchSequence matchSequence : matchSequences) {
                        defaultDecision.addAuthority(matchSequence.getTokenPattern().getName());
                    }
                    TaggedToken<TokeniserOutcome> defaultTaggedToken = new TaggedToken<>(token, defaultDecision, TokeniserOutcome.valueOf(defaultDecision.getOutcome()));
                    TokenisedAtomicTokenSequence defaultSequence = new TokenisedAtomicTokenSequence(history);
                    defaultSequence.add(defaultTaggedToken);
                    defaultSequence.addDecision(defaultDecision);
                    heap.add(defaultSequence);
                    // Apply one non-default decision per match sequence
                    for (int k = 0; k < matchSequences.size(); k++) {
                        TokenPatternMatchSequence matchSequence = matchSequences.get(k);
                        double prob = decisionProbs[k + 1];
                        Decision decision = new Decision(otherOutcome.name(), prob);
                        decision.addAuthority("_" + this.getClass().getSimpleName());
                        decision.addAuthority("_" + "Patterns");
                        decision.addAuthority(matchSequence.getTokenPattern().getName());
                        TaggedToken<TokeniserOutcome> taggedToken = new TaggedToken<>(token, decision, TokeniserOutcome.valueOf(decision.getOutcome()));
                        TokenisedAtomicTokenSequence newSequence = new TokenisedAtomicTokenSequence(history);
                        newSequence.add(taggedToken);
                        newSequence.addDecision(decision);
                        // in this sequence to the solution
                        for (Token tokenInSequence : matchSequence.getTokensToCheck()) {
                            if (tokenInSequence.equals(token)) {
                                continue;
                            }
                            Decision decisionInSequence = new Decision(decision.getOutcome());
                            decisionInSequence.addAuthority("_" + this.getClass().getSimpleName());
                            decisionInSequence.addAuthority("_" + "DecisionInSequence");
                            decisionInSequence.addAuthority("_" + "DecisionInSequence_non_default");
                            decisionInSequence.addAuthority("_" + "Patterns");
                            TaggedToken<TokeniserOutcome> taggedTokenInSequence = new TaggedToken<>(tokenInSequence, decisionInSequence, TokeniserOutcome.valueOf(decisionInSequence.getOutcome()));
                            newSequence.add(taggedTokenInSequence);
                        }
                        heap.add(newSequence);
                    }
                // next sequence
                } else {
                    // token doesn't start match sequence, and hasn't
                    // already been added to the current sequence
                    Decision decision = defaultDecisions.get(i);
                    if (matchedTokens.contains(token)) {
                        decision = new Decision(decision.getOutcome());
                        decision.addAuthority("_" + this.getClass().getSimpleName());
                        decision.addAuthority("_" + "DecisionInSequence");
                        decision.addAuthority("_" + "DecisionInSequence_default");
                        decision.addAuthority("_" + "Patterns");
                    }
                    TaggedToken<TokeniserOutcome> taggedToken = new TaggedToken<>(token, decision, TokeniserOutcome.valueOf(decision.getOutcome()));
                    TokenisedAtomicTokenSequence newSequence = new TokenisedAtomicTokenSequence(history);
                    newSequence.add(taggedToken);
                    heap.add(newSequence);
                }
            }
        // next sequence in the old heap
        }
        // next token
        sequences = new ArrayList<TokenisedAtomicTokenSequence>();
        int k = 0;
        while (!heap.isEmpty()) {
            sequences.add(heap.poll());
            k++;
            if (k >= this.getBeamWidth())
                break;
        }
    } else {
        sequences = new ArrayList<TokenisedAtomicTokenSequence>();
        TokenisedAtomicTokenSequence defaultSequence = new TokenisedAtomicTokenSequence(sentence, 0, this.getSessionId());
        int i = 0;
        for (Token token : initialSequence.listWithWhiteSpace()) {
            Decision decision = defaultDecisions.get(i++);
            TaggedToken<TokeniserOutcome> taggedToken = new TaggedToken<>(token, decision, TokeniserOutcome.valueOf(decision.getOutcome()));
            defaultSequence.add(taggedToken);
        }
        sequences.add(defaultSequence);
    }
    // have decision maker?
    return sequences;
}

Example 10 with FeatureResult

use of com.joliciel.talismane.machineLearning.features.FeatureResult in project talismane by joliciel-informatique.

the class LinearSVMUtils method prepareData.

public static List<Feature> prepareData(List<FeatureResult<?>> featureResults, TObjectIntMap<String> featureIndexMap) {
    List<Feature> featureList = new ArrayList<Feature>(featureResults.size());
    for (FeatureResult<?> featureResult : featureResults) {
        if (featureResult.getOutcome() instanceof List) {
            @SuppressWarnings("unchecked") FeatureResult<List<WeightedOutcome<String>>> stringCollectionResult = (FeatureResult<List<WeightedOutcome<String>>>) featureResult;
            for (WeightedOutcome<String> stringOutcome : stringCollectionResult.getOutcome()) {
                int index = featureIndexMap.get(featureResult.getTrainingName() + "|" + featureResult.getTrainingOutcome(stringOutcome.getOutcome()));
                if (index >= 0) {
                    double value = stringOutcome.getWeight();
                    FeatureNode featureNode = new FeatureNode(index, value);
                    featureList.add(featureNode);
                }
            }
        } else {
            double value = 1.0;
            if (featureResult.getOutcome() instanceof Double) {
                @SuppressWarnings("unchecked") FeatureResult<Double> doubleResult = (FeatureResult<Double>) featureResult;
                value = doubleResult.getOutcome().doubleValue();
            }
            int index = featureIndexMap.get(featureResult.getTrainingName());
            if (index >= 0) {
                // we only need to bother adding features which existed in the
                // training set
                FeatureNode featureNode = new FeatureNode(index, value);
                featureList.add(featureNode);
            }
        }
    }
    return featureList;
}