Examples with Datum edu.stanford.nlp.ling.Datum used on opensource projects

Example 1 with Datum

use of edu.stanford.nlp.ling.Datum in project CoreNLP by stanfordnlp.

the class ChineseMaxentLexicon method ensureProbs.

private void ensureProbs(int word, boolean subtractTagScore) {
    if (word == lastWord) {
        return;
    }
    lastWord = word;
    if (functionWordTags.containsKey(wordIndex.get(word))) {
        logProbs = new ClassicCounter<>();
        String trueTag = functionWordTags.get(wordIndex.get(word));
        for (String tag : tagIndex.objectsList()) {
            if (ctlp.basicCategory(tag).equals(trueTag)) {
                logProbs.setCount(tag, 0);
            } else {
                logProbs.setCount(tag, Double.NEGATIVE_INFINITY);
            }
        }
        return;
    }
    Datum datum = new BasicDatum(featExtractor.makeFeatures(wordIndex.get(word)));
    logProbs = scorer.logProbabilityOf(datum);
    if (subtractTagScore) {
        Set<String> tagSet = logProbs.keySet();
        for (String tag : tagSet) {
            logProbs.incrementCount(tag, -Math.log(tagDist.probabilityOf(tag)));
        }
    }
}