use of edu.illinois.cs.cogcomp.lbjava.parse.FoldParser in project cogcomp-nlp by CogComp.
the class ClassifierComparison method printConstrainedClassifierPerformance.
public static void printConstrainedClassifierPerformance(Parser parser) {
List<Pair<Classifier, EvaluateDiscrete>> classifiers = new ArrayList<>();
LocalCommaClassifier learner = new LocalCommaClassifier();
EvaluateDiscrete unconstrainedPerformance = new EvaluateDiscrete();
learner.setLTU(new SparseAveragedPerceptron(0.003, 0, 3.5));
classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new SubstitutePairConstrainedCommaClassifier(), new EvaluateDiscrete()));
classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new LocativePairConstrainedCommaClassifier(), new EvaluateDiscrete()));
classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new ListCommasConstrainedCommaClassifier(), new EvaluateDiscrete()));
classifiers.add(new Pair<Classifier, EvaluateDiscrete>(new OxfordCommaConstrainedCommaClassifier(), new EvaluateDiscrete()));
int k = 5;
parser.reset();
FoldParser foldParser = new FoldParser(parser, k, SplitPolicy.sequential, 0, false);
for (int i = 0; i < k; foldParser.setPivot(++i)) {
foldParser.setFromPivot(false);
foldParser.reset();
learner.forget();
BatchTrainer bt = new BatchTrainer(learner, foldParser);
Lexicon lexicon = bt.preExtract(null);
learner.setLexicon(lexicon);
bt.train(250);
learner.save();
foldParser.setFromPivot(true);
foldParser.reset();
unconstrainedPerformance.reportAll(EvaluateDiscrete.evaluateDiscrete(learner, learner.getLabeler(), foldParser));
for (Pair<Classifier, EvaluateDiscrete> pair : classifiers) {
foldParser.reset();
pair.getSecond().reportAll(EvaluateDiscrete.evaluateDiscrete(pair.getFirst(), learner.getLabeler(), foldParser));
}
}
for (Pair<Classifier, EvaluateDiscrete> pair : classifiers) {
System.out.println(pair.getFirst().name + " " + pair.getSecond().getOverallStats()[2]);
}
}
use of edu.illinois.cs.cogcomp.lbjava.parse.FoldParser in project cogcomp-nlp by CogComp.
the class ClassifierComparison method localCVal.
public static EvaluateDiscrete localCVal(boolean trainOnGold, boolean testOnGold, Parser parser, int learningRounds, double learningRate, double threshold, double thickness, boolean testOnTrain) {
int k = 5;
LocalCommaClassifier learner = new LocalCommaClassifier();
learner.setLTU(new SparseAveragedPerceptron(learningRate, threshold, thickness));
parser.reset();
final FoldParser foldParser = new FoldParser(parser, k, SplitPolicy.sequential, 0, false);
EvaluateDiscrete performanceRecord = new EvaluateDiscrete();
for (int i = 0; i < k; foldParser.setPivot(++i)) {
foldParser.setFromPivot(false);
foldParser.reset();
learner.forget();
BatchTrainer bt = new BatchTrainer(learner, foldParser);
Comma.useGoldFeatures(trainOnGold);
Lexicon lexicon = bt.preExtract(null);
learner.setLexicon(lexicon);
bt.train(learningRounds);
if (!testOnTrain)
foldParser.setFromPivot(true);
foldParser.reset();
Comma.useGoldFeatures(testOnGold);
EvaluateDiscrete currentPerformance = EvaluateDiscrete.evaluateDiscrete(learner, learner.getLabeler(), foldParser);
performanceRecord.reportAll(currentPerformance);
}
// System.out.println(performanceRecord.getOverallStats()[2]);
performanceRecord.printPerformance(System.out);
// performanceRecord.printConfusion(System.out);
return performanceRecord;
}
use of edu.illinois.cs.cogcomp.lbjava.parse.FoldParser in project cogcomp-nlp by CogComp.
the class ClassifierComparison method structuredCVal.
public static EvaluateDiscrete structuredCVal(StructuredCommaClassifier model, Parser parser, boolean useGoldFeatures, boolean testOnTrain) throws Exception {
Comma.useGoldFeatures(useGoldFeatures);
int k = 5;
parser.reset();
FoldParser foldParser = new FoldParser(parser, k, SplitPolicy.sequential, 0, false);
EvaluateDiscrete cvalResult = new EvaluateDiscrete();
for (int i = 0; i < k; foldParser.setPivot(++i)) {
foldParser.setFromPivot(false);
foldParser.reset();
LinkedHashSet<CommaSRLSentence> trainSentences = new LinkedHashSet<>();
for (Object comma = foldParser.next(); comma != null; comma = foldParser.next()) {
trainSentences.add(((Comma) comma).getSentence());
}
model.train(new ArrayList<>(trainSentences), null);
if (!testOnTrain)
foldParser.setFromPivot(true);
foldParser.reset();
LinkedHashSet<CommaSRLSentence> testSentences = new LinkedHashSet<>();
for (Object comma = foldParser.next(); comma != null; comma = foldParser.next()) {
testSentences.add(((Comma) comma).getSentence());
}
EvaluateDiscrete evaluator = model.test(new ArrayList<>(testSentences), null);
cvalResult.reportAll(evaluator);
}
cvalResult.printPerformance(System.out);
return cvalResult;
}
Aggregations