use of edu.illinois.cs.cogcomp.lbjava.parse.Parser in project cogcomp-nlp by CogComp.
the class TestDiff method testDiff.
@Test
public void testDiff() {
Chunker tagger = new Chunker();
Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(testFile)));
String previous = "";
String sentence = "";
int sentenceCounter = 0;
for (Token w = (Token) parser.next(); w != null; w = (Token) parser.next()) {
String prediction = tagger.discreteValue(w);
if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
sentence += ("[" + prediction.substring(2) + " ");
sentence += ("(" + w.partOfSpeech + " " + w.form + ") ");
if (!prediction.equals("O") && (w.next == null || tagger.discreteValue(w.next).equals("O") || tagger.discreteValue(w.next).startsWith("B-") || !tagger.discreteValue(w.next).endsWith(prediction.substring(2))))
sentence += ("] ");
if (w.next == null) {
sentence = sentence.trim();
String refSentence = refSentences.get(sentenceCounter).trim();
if (!sentence.equals(refSentence))
fail("Produced output doesn't match reference: " + "\nProduced: " + sentence + "\nExpected: " + refSentence);
sentence = "";
sentenceCounter++;
}
previous = prediction;
}
}
use of edu.illinois.cs.cogcomp.lbjava.parse.Parser in project cogcomp-nlp by CogComp.
the class ChunkerTrain method trainModels.
/**
* Trains the chunker models with the specified training data which must be in CoNLL2000 format
*
* @param trainingData The labeled training data
*/
public void trainModels(String trainingData) {
Parser parser = new CoNLL2000Parser(trainingData);
trainModelsWithParser(parser);
}
use of edu.illinois.cs.cogcomp.lbjava.parse.Parser in project cogcomp-nlp by CogComp.
the class ChunkerTrain method trainModels.
public void trainModels(String trainingData, String modeldir, String modelname, double dev_ratio) {
Parser parser = new CoNLL2000Parser(trainingData);
trainModelsWithParser(parser, modeldir, modelname, dev_ratio);
}
use of edu.illinois.cs.cogcomp.lbjava.parse.Parser in project cogcomp-nlp by CogComp.
the class ChunksAndPOSTags method main.
public static void main(String[] args) {
String filename = null;
try {
filename = args[0];
if (args.length > 1)
throw new Exception();
} catch (Exception e) {
System.err.println("usage: java edu.illinois.cs.cogcomp.chunker.main.ChunksAndPOSTags <input file>");
System.exit(1);
}
Chunker chunker = new Chunker();
Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(filename)));
String previous = "";
for (Word w = (Word) parser.next(); w != null; w = (Word) parser.next()) {
String prediction = chunker.discreteValue(w);
if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
logger.info("[" + prediction.substring(2) + " ");
logger.info("(" + w.partOfSpeech + " " + w.form + ") ");
if (!prediction.equals("O") && (w.next == null || chunker.discreteValue(w.next).equals("O") || chunker.discreteValue(w.next).startsWith("B-") || !chunker.discreteValue(w.next).endsWith(prediction.substring(2))))
logger.info("] ");
if (w.next == null)
logger.info("\n");
previous = prediction;
}
}
use of edu.illinois.cs.cogcomp.lbjava.parse.Parser in project cogcomp-nlp by CogComp.
the class TestChunkerModels method testAccuracy.
public void testAccuracy() {
Parser parser = new ChildrenFromVectors(new CoNLL2000Parser(labeledData));
int numSeen = 0;
int numEqual = 0;
for (Token w = (Token) parser.next(); w != null; w = (Token) parser.next()) {
String prediction = tagger.discreteValue(w);
String raw = w.toString();
String actualChunk = raw.substring(raw.indexOf('(') + 1, raw.indexOf(' '));
if (prediction.equals(actualChunk)) {
numEqual++;
}
numSeen++;
}
logger.info("Total accuracy over " + numSeen + " items: " + String.format("%.2f", 100.0 * (double) numEqual / (double) numSeen) + "%");
}
Aggregations