Examples with ScoreSet edu.illinois.cs.cogcomp.lbjava.classify.ScoreSet used on opensource projects

Example 1 with ScoreSet

use of edu.illinois.cs.cogcomp.lbjava.classify.ScoreSet in project cogcomp-nlp by CogComp.

the class ACERelationTester method test_cv_gold.

/*
     * This function only tests the constrained classifier
     * It performs a similar five-fold cv
     */
public static void test_cv_gold() {
    int total_correct = 0;
    int total_labeled = 0;
    int total_predicted = 0;
    int total_coarse_correct = 0;
    for (int i = 0; i < 5; i++) {
        fine_relation_label output = new fine_relation_label();
        ACEMentionReader train_parser = IOHelper.readFiveFold(i, "TRAIN");
        relation_classifier classifier = new relation_classifier();
        classifier.setLexiconLocation("models/relation_classifier_fold_" + i + ".lex");
        BatchTrainer trainer = new BatchTrainer(classifier, train_parser);
        Learner preExtractLearner = trainer.preExtract("models/relation_classifier_fold_" + i + ".ex", true, Lexicon.CountPolicy.none);
        preExtractLearner.saveLexicon();
        Lexicon lexicon = preExtractLearner.getLexicon();
        classifier.setLexicon(lexicon);
        int examples = train_parser.relations_bi.size();
        classifier.initialize(examples, preExtractLearner.getLexicon().size());
        for (Relation r : train_parser.relations_bi) {
            classifier.learn(r);
        }
        classifier.doneWithRound();
        classifier.doneLearning();
        ACERelationConstrainedClassifier constrainedClassifier = new ACERelationConstrainedClassifier(classifier);
        ACEMentionReader test_parser = IOHelper.readFiveFold(i, "TEST");
        for (Relation r : test_parser.relations_bi) {
            String predicted_label = constrainedClassifier.discreteValue(r);
            String gold_label = output.discreteValue(r);
            Relation oppoR = new Relation("TO_TEST", r.getTarget(), r.getSource(), 1.0f);
            String oppo_predicted_label = constrainedClassifier.discreteValue(oppoR);
            if (!predicted_label.equals(ACEMentionReader.getOppoName(oppo_predicted_label))) {
                ScoreSet scores = classifier.scores(r);
                Score[] scoresArray = scores.toArray();
                double score_curtag = 0.0;
                for (Score score : scoresArray) {
                    if (score.value.equals(predicted_label)) {
                        score_curtag = score.score;
                    }
                }
                scores = classifier.scores((Object) oppoR);
                scoresArray = scores.toArray();
                double oppo_score_opptag = 0.0;
                for (Score score : scoresArray) {
                    if (score.value.equals(oppo_predicted_label)) {
                        oppo_score_opptag = score.score;
                    }
                }
                if (score_curtag < oppo_score_opptag && oppo_score_opptag - score_curtag > 0.005) {
                    predicted_label = ACEMentionReader.getOppoName(oppo_predicted_label);
                }
            }
            if (!predicted_label.equals("NOT_RELATED")) {
                total_predicted++;
            }
            if (!gold_label.equals("NOT_RELATED")) {
                total_labeled++;
            }
            if (predicted_label.equals(gold_label)) {
                if (!predicted_label.equals("NOT_RELATED")) {
                    total_correct++;
                }
            }
            if (getCoarseType(predicted_label).equals(getCoarseType(gold_label))) {
                if (!predicted_label.equals("NOT_RELATED")) {
                    total_coarse_correct++;
                }
            }
        }
        classifier.forget();
    }
    System.out.println("Total labeled: " + total_labeled);
    System.out.println("Total predicted: " + total_predicted);
    System.out.println("Total correct: " + total_correct);
    System.out.println("Total coarse correct: " + total_coarse_correct);
    double p = (double) total_correct * 100.0 / (double) total_predicted;
    double r = (double) total_correct * 100.0 / (double) total_labeled;
    double f = 2 * p * r / (p + r);
    System.out.println("Precision: " + p);
    System.out.println("Recall: " + r);
    System.out.println("Fine Type F1: " + f);
    System.out.println("Coarse Type F1: " + f * (double) total_coarse_correct / (double) total_correct);
}

Example 2 with ScoreSet

use of edu.illinois.cs.cogcomp.lbjava.classify.ScoreSet in project cogcomp-nlp by CogComp.

the class RelationAnnotator method addView.

@Override
public void addView(TextAnnotation record) throws AnnotatorException {
    if (!isInitialized()) {
        doInitialize();
    }
    if (!record.hasView(ViewNames.POS)) {
        throw new AnnotatorException("Missing required view POS");
    }
    if (!record.hasView(ViewNames.DEPENDENCY_STANFORD)) {
        throw new AnnotatorException("Missing required view DEPENDENCY_STANFORD");
    }
    if (!record.hasView(ViewNames.SHALLOW_PARSE)) {
        throw new AnnotatorException("Missing required view SHALLOW_PARSE");
    }
    if (!record.hasView(ViewNames.MENTION)) {
        throw new AnnotatorException("Missing required view MENTION");
    }
    View mentionView = record.getView(ViewNames.MENTION);
    View relationView = new SpanLabelView(ViewNames.RELATION, record);
    // Add the original mention view if no mentions are predicted.
    if (mentionView.getConstituents().size() == 0) {
        record.addView(ViewNames.RELATION, relationView);
        return;
    }
    if (mentionView.getConstituents().get(0).getAttribute("EntityType").equals("MENTION")) {
        logger.error("The mentions don't have types; this will cause poor performance in predictions.. . ");
    }
    View annotatedTokenView = new SpanLabelView("RE_ANNOTATED", record);
    for (Constituent co : record.getView(ViewNames.TOKENS).getConstituents()) {
        Constituent c = co.cloneForNewView("RE_ANNOTATED");
        for (String s : co.getAttributeKeys()) {
            c.addAttribute(s, co.getAttribute(s));
        }
        c.addAttribute("WORDNETTAG", BIOFeatureExtractor.getWordNetTags(wordNet, c));
        c.addAttribute("WORDNETHYM", BIOFeatureExtractor.getWordNetHyms(wordNet, c));
        annotatedTokenView.addConstituent(c);
    }
    record.addView("RE_ANNOTATED", annotatedTokenView);
    for (int i = 0; i < record.getNumberOfSentences(); i++) {
        Sentence curSentence = record.getSentence(i);
        List<Constituent> cins = mentionView.getConstituentsCoveringSpan(curSentence.getStartSpan(), curSentence.getEndSpan());
        for (int j = 0; j < cins.size(); j++) {
            for (int k = j + 1; k < cins.size(); k++) {
                if (k == j)
                    continue;
                Constituent source = cins.get(j);
                Constituent target = cins.get(k);
                Constituent sourceHead = MentionAnnotator.getHeadConstituent(source, "");
                Constituent targetHead = MentionAnnotator.getHeadConstituent(target, "");
                source.addAttribute("GAZ", ((FlatGazetteers) gazetteers).annotatePhrase(sourceHead));
                target.addAttribute("GAZ", ((FlatGazetteers) gazetteers).annotatePhrase(targetHead));
                Relation for_test_forward = new Relation("PredictedRE", source, target, 1.0f);
                Relation for_test_backward = new Relation("PredictedRE", target, source, 1.0f);
                String tag_forward = constrainedClassifier.discreteValue(for_test_forward);
                String tag_backward = constrainedClassifier.discreteValue(for_test_backward);
                if (tag_forward.equals(ACEMentionReader.getOppoName(tag_backward)) && !tag_forward.equals("NOT_RELATED")) {
                    String tag = tag_forward;
                    Constituent first = source;
                    Constituent second = target;
                    if (tag_forward.length() > tag_backward.length()) {
                        tag = tag_backward;
                        first = target;
                        second = source;
                    }
                    String coarseType = ACERelationTester.getCoarseType(tag);
                    Constituent firstMention = first.cloneForNewView(ViewNames.RELATION);
                    Constituent secondMention = second.cloneForNewView(ViewNames.RELATION);
                    Relation r = new Relation(coarseType + "-" + tag, firstMention, secondMention, 1.0f);
                    r.addAttribute("RelationType", coarseType);
                    r.addAttribute("RelationSubtype", tag);
                    relationView.addConstituent(firstMention);
                    relationView.addConstituent(secondMention);
                    relationView.addRelation(r);
                }
                if (!tag_forward.equals(ACEMentionReader.getOppoName(tag_backward)) && (!tag_forward.equals("NOT_RELATED") || !tag_backward.equals("NOT_RELATED"))) {
                    double forward_score = 0.0;
                    double backward_score = 0.0;
                    ScoreSet scores = relationClassifier.scores(for_test_forward);
                    Score[] scoresArray = scores.toArray();
                    for (Score s : scoresArray) {
                        if (s.value.equals(tag_forward)) {
                            forward_score = s.score;
                        }
                    }
                    scores = relationClassifier.scores(for_test_backward);
                    scoresArray = scores.toArray();
                    for (Score s : scoresArray) {
                        if (s.value.equals(tag_forward)) {
                            backward_score = s.score;
                        }
                    }
                    String tag = tag_forward;
                    Constituent first = source;
                    Constituent second = target;
                    if (forward_score < backward_score && backward_score - forward_score > 0.005) {
                        tag = tag_backward;
                        first = target;
                        second = source;
                    }
                    if (!tag.equals("NOT_RELATED")) {
                        Constituent firstMention = first.cloneForNewView(ViewNames.RELATION);
                        Constituent secondMention = second.cloneForNewView(ViewNames.RELATION);
                        Relation r;
                        String coarseType = ACERelationTester.getCoarseType(tag);
                        if (tag.contains("_OP")) {
                            tag = ACEMentionReader.getOppoName(tag);
                            r = new Relation(coarseType + "-" + tag, secondMention, firstMention, 1.0f);
                        } else {
                            r = new Relation(coarseType + "-" + tag, firstMention, secondMention, 1.0f);
                        }
                        r.addAttribute("RelationType", coarseType);
                        r.addAttribute("RelationSubtype", tag);
                        relationView.addConstituent(firstMention);
                        relationView.addConstituent(secondMention);
                        relationView.addRelation(r);
                    }
                }
            }
        }
    }
    record.addView(ViewNames.RELATION, relationView);
}

Example 3 with ScoreSet

use of edu.illinois.cs.cogcomp.lbjava.classify.ScoreSet in project cogcomp-nlp by CogComp.

the class BIOTester method joint_inference.

/**
 * @param t The target Consitutent
 * @param candidates The learner array containing 3 Learners.
 *                    candidates[0] : NAM
 *                    candidates[1] : NOM
 *                    candidates[2] : PRO
 * @return a pair of a String and a Integer.
 *          The String: The result of the joint inferencing
 *          The Integer: The index of the selected learner in candidates
 */
public static Pair<String, Integer> joint_inference(Constituent t, Learner[] candidates) {
    double highest_start_score = -10.0;
    Map<Integer, Double> remaining = new ConcurrentHashMap<>();
    String[] preBIOLevel1 = new String[3];
    String[] preBIOLevel2 = new String[3];
    for (int i = 0; i < 3; i++) {
        preBIOLevel2[i] = "O";
    }
    int chosen = -1;
    for (int i = 0; i < candidates.length; i++) {
        if (candidates[i] == null) {
            continue;
        }
        String prediction = candidates[i].discreteValue(t);
        preBIOLevel1[i] = prediction;
        if (prediction.startsWith("B") || prediction.startsWith("U")) {
            ScoreSet scores = candidates[i].scores(t);
            Score[] scoresArray = scores.toArray();
            for (Score s : scoresArray) {
                if (s.value.equals(prediction)) {
                    remaining.put(i, s.score);
                    if (s.score > highest_start_score) {
                        highest_start_score = s.score;
                        chosen = i;
                    }
                }
            }
        }
    }
    if (chosen == -1) {
        return new Pair<>("O", -1);
    } else {
        return new Pair<>(candidates[chosen].discreteValue(t), chosen);
    }
}