Examples with Lexicon edu.illinois.cs.cogcomp.lbjava.learn.Lexicon used on opensource projects

Example 1 with Lexicon

use of edu.illinois.cs.cogcomp.lbjava.learn.Lexicon in project cogcomp-nlp by CogComp.

the class POSTagger method getTagValues.

public Set<String> getTagValues() {
    Lexicon labelLexicon = taggerKnown.getLabelLexicon();
    Set<String> tagSet = new HashSet();
    for (int i = 0; i < labelLexicon.size(); ++i) {
        tagSet.add(labelLexicon.lookupKey(i).getStringValue());
    }
    return tagSet;
}

Example 2 with Lexicon

use of edu.illinois.cs.cogcomp.lbjava.learn.Lexicon in project cogcomp-nlp by CogComp.

the class NERAnnotator method getTagValues.

/**
     * Return possible tag values that the NERAnnotator can produce.
     *
     * @return the set of string representing the tag values
     */
@Override
public Set<String> getTagValues() {
    if (!isInitialized()) {
        doInitialize();
    }
    Lexicon labelLexicon = t1.getLabelLexicon();
    Set<String> tagSet = new HashSet<String>();
    for (int i = 0; i < labelLexicon.size(); ++i) {
        tagSet.add(labelLexicon.lookupKey(i).getStringValue());
    }
    return tagSet;
}

Example 3 with Lexicon

use of edu.illinois.cs.cogcomp.lbjava.learn.Lexicon in project cogcomp-nlp by CogComp.

the class ClassifierComparison method printConstrainedClassifierPerformance.

public static void printConstrainedClassifierPerformance(Parser parser) {
    List<Pair<Classifier, EvaluateDiscrete>> classifiers = new ArrayList<>();
    LocalCommaClassifier learner = new LocalCommaClassifier();
    EvaluateDiscrete unconstrainedPerformance = new EvaluateDiscrete();
    learner.setLTU(new SparseAveragedPerceptron(0.003, 0, 3.5));
    classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new SubstitutePairConstrainedCommaClassifier(), new EvaluateDiscrete()));
    classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new LocativePairConstrainedCommaClassifier(), new EvaluateDiscrete()));
    classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new ListCommasConstrainedCommaClassifier(), new EvaluateDiscrete()));
    classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new OxfordCommaConstrainedCommaClassifier(), new EvaluateDiscrete()));
    int k = 5;
    parser.reset();
    FoldParser foldParser = new FoldParser(parser, k, SplitPolicy.sequential, 0, false);
    for (int i = 0; i < k; foldParser.setPivot(++i)) {
        foldParser.setFromPivot(false);
        foldParser.reset();
        learner.forget();
        BatchTrainer bt = new BatchTrainer(learner, foldParser);
        Lexicon lexicon = bt.preExtract(null);
        learner.setLexicon(lexicon);
        bt.train(250);
        learner.save();
        foldParser.setFromPivot(true);
        foldParser.reset();
        unconstrainedPerformance.reportAll(EvaluateDiscrete.evaluateDiscrete(learner, learner.getLabeler(), foldParser));
        for (Pair<Classifier, EvaluateDiscrete> pair : classifiers) {
            foldParser.reset();
            pair.getSecond().reportAll(EvaluateDiscrete.evaluateDiscrete(pair.getFirst(), learner.getLabeler(), foldParser));
        }
    }
    for (Pair<Classifier, EvaluateDiscrete> pair : classifiers) {
        System.out.println(pair.getFirst().name + " " + pair.getSecond().getOverallStats()[2]);
    }
}

Example 4 with Lexicon

use of edu.illinois.cs.cogcomp.lbjava.learn.Lexicon in project cogcomp-nlp by CogComp.

the class ClassifierComparison method localCVal.

public static EvaluateDiscrete localCVal(boolean trainOnGold, boolean testOnGold, Parser parser, int learningRounds, double learningRate, double threshold, double thickness, boolean testOnTrain) {
    int k = 5;
    LocalCommaClassifier learner = new LocalCommaClassifier();
    learner.setLTU(new SparseAveragedPerceptron(learningRate, threshold, thickness));
    parser.reset();
    final FoldParser foldParser = new FoldParser(parser, k, SplitPolicy.sequential, 0, false);
    EvaluateDiscrete performanceRecord = new EvaluateDiscrete();
    for (int i = 0; i < k; foldParser.setPivot(++i)) {
        foldParser.setFromPivot(false);
        foldParser.reset();
        learner.forget();
        BatchTrainer bt = new BatchTrainer(learner, foldParser);
        Comma.useGoldFeatures(trainOnGold);
        Lexicon lexicon = bt.preExtract(null);
        learner.setLexicon(lexicon);
        bt.train(learningRounds);
        if (!testOnTrain)
            foldParser.setFromPivot(true);
        foldParser.reset();
        Comma.useGoldFeatures(testOnGold);
        EvaluateDiscrete currentPerformance = EvaluateDiscrete.evaluateDiscrete(learner, learner.getLabeler(), foldParser);
        performanceRecord.reportAll(currentPerformance);
    }
    // System.out.println(performanceRecord.getOverallStats()[2]);
    performanceRecord.printPerformance(System.out);
    // performanceRecord.printConfusion(System.out);
    return performanceRecord;
}

Example 5 with Lexicon

use of edu.illinois.cs.cogcomp.lbjava.learn.Lexicon in project cogcomp-nlp by CogComp.

the class ChunkerAnnotator method getTagValues.

/**
     * Return possible tag values that the ChunkerAnnotator can produce.
     *
     * @return the set of string representing the tag values
     */
@Override
public Set<String> getTagValues() {
    if (!isInitialized()) {
        doInitialize();
    }
    Lexicon labelLexicon = tagger.getLabelLexicon();
    Set<String> tagSet = new HashSet();
    for (int i = 0; i < labelLexicon.size(); ++i) {
        tagSet.add(labelLexicon.lookupKey(i).getStringValue());
    }
    return tagSet;
}