use of edu.illinois.cs.cogcomp.lbjava.nlp.Word in project cogcomp-nlp by CogComp.
the class ColumnFileReader method next.
public Object next() {
String token = null;
String pos = null;
String label = null;
linec++;
// Skip to start of next line, skip unnecessary blank lines, headers and so on.
String[] line = (String[]) super.next();
while (line != null && (line.length == 0 || (line.length > 4 && line[4].equals("-X-")))) {
line = (String[]) super.next();
linec++;
}
if (line == null)
return null;
// parse the data, CoNLL 2002 or CoNLL 2003.
if (line.length == 2) {
token = line[0];
label = line[1];
} else {
token = line[5];
label = line[0];
pos = line[4];
}
LinkedVector res = new LinkedVector();
NEWord w = new NEWord(new Word(token, pos), null, label);
NEWord.addTokenToSentence(res, w.form, w.neLabel);
for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) {
linec++;
// parse the data, CoNLL 2002 or CoNLL 2003.
if (line.length == 2) {
token = line[0];
label = line[1];
} else if (line.length > 5) {
token = line[5];
label = line[0];
pos = line[4];
} else {
System.out.println("Line " + linec + " in " + filename + " is wrong with " + line.length);
for (String a : line) System.out.print(":" + a);
System.out.println();
continue;
}
w = new NEWord(new Word(token, pos), null, label);
NEWord.addTokenToSentence(res, w.form, w.neLabel);
}
if (res.size() == 0)
return null;
return res;
}
use of edu.illinois.cs.cogcomp.lbjava.nlp.Word in project cogcomp-nlp by CogComp.
the class IllinoisTokenizer method tokenizeTextSpan.
/**
* given a span of text, return a list of {@literal Pair< String[], IntPair[] >} corresponding
* to tokenized sentences, where the String[] is the ordered list of sentence tokens and the
* IntPair[] is the corresponding list of character offsets with respect to <b>the original
* text</b>.
*
* @param text an arbitrary span of text.
* @return a {@link Tokenization} object containing the ordered token strings, their character
* offsets, and sentence end positions (as one-past-the-end token offsets)
*/
@Override
public Tokenization tokenizeTextSpan(String text) {
String[] splitterInput = new String[1];
splitterInput[0] = text;
SentenceSplitter splitter = new SentenceSplitter(splitterInput);
Sentence[] sentences = splitter.splitAll();
List<IntPair> characterOffsets = new LinkedList<>();
int[] sentenceEndTokenOffsets = new int[sentences.length];
int sentenceEndTokenIndex = 0;
int sentIndex = 0;
List<String> tokens = new LinkedList<>();
for (Sentence s : splitter.splitAll()) {
LinkedVector words = s.wordSplit();
if (s.end >= text.length()) {
throw new IllegalArgumentException("Error in tokenizer, sentence end ( " + s.end + ") is greater than rawtext length (" + text.length() + ").");
}
for (int i = 0; i < words.size(); i++) {
Word word = (Word) words.get(i);
IntPair wordOffsets = new IntPair(word.start, word.end + 1);
characterOffsets.add(wordOffsets);
tokens.add(text.substring(wordOffsets.getFirst(), wordOffsets.getSecond()));
}
sentenceEndTokenIndex += words.size();
sentenceEndTokenOffsets[sentIndex++] = sentenceEndTokenIndex;
}
String[] tokenArray = tokens.toArray(new String[tokens.size()]);
IntPair[] charOffsetArray = characterOffsets.toArray(new IntPair[characterOffsets.size()]);
return new Tokenization(tokenArray, charOffsetArray, sentenceEndTokenOffsets);
}
use of edu.illinois.cs.cogcomp.lbjava.nlp.Word in project cogcomp-nlp by CogComp.
the class POSTag method main.
/**
* Implements the program described above.
*
* @param args The command line parameters.
**/
public static void main(String[] args) {
// Parse the command line
if (!(args.length == 1 && !args[0].startsWith("-") || args.length == 2 && (args[0].equals("-q") || args[0].equals("--quiet")) && !args[1].startsWith("-"))) {
System.err.println("usage: java edu.illinois.cs.cogcomp.lbj.pos.POSTag [-q] <testing set>\n" + " If -q is specified, the only output is timing and accuracy\n" + " information. Otherwise, the testing set is output with\n" + " extra tags indicating whether each prediction was correct.");
System.exit(1);
}
boolean quiet = args.length == 2;
testingFile = args[args.length - 1];
POSTagger tagger = new POSTagger();
BufferedReader in = open();
int correct = 0, incorrect = 0;
for (String line = readLine(in); line != null; line = readLine(in)) {
LinkedVector sentence = POSBracketToVector.parsePOSBracketForm(line);
for (Word word = (Word) sentence.get(0); word != null; word = (Word) word.next) {
String label = word.partOfSpeech;
word.partOfSpeech = null;
String prediction = tagger.discreteValue(word);
if (prediction.equals(label)) {
++correct;
if (!quiet)
System.out.print("+");
} else {
++incorrect;
if (!quiet)
System.out.print("-[" + label + "]");
}
if (!quiet)
System.out.print(word + " ");
}
if (!quiet)
System.out.print("");
}
System.out.println("Accuracy: " + (100 * correct / (double) (correct + incorrect)) + "%");
}
use of edu.illinois.cs.cogcomp.lbjava.nlp.Word in project cogcomp-nlp by CogComp.
the class ChunksAndPOSTags method main.
public static void main(String[] args) {
String filename = null;
try {
filename = args[0];
if (args.length > 1)
throw new Exception();
} catch (Exception e) {
System.err.println("usage: java edu.illinois.cs.cogcomp.chunker.main.ChunksAndPOSTags <input file>");
System.exit(1);
}
Chunker chunker = new Chunker();
Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(filename)));
String previous = "";
for (Word w = (Word) parser.next(); w != null; w = (Word) parser.next()) {
String prediction = chunker.discreteValue(w);
if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
logger.info("[" + prediction.substring(2) + " ");
logger.info("(" + w.partOfSpeech + " " + w.form + ") ");
if (!prediction.equals("O") && (w.next == null || chunker.discreteValue(w.next).equals("O") || chunker.discreteValue(w.next).startsWith("B-") || !chunker.discreteValue(w.next).endsWith(prediction.substring(2))))
logger.info("] ");
if (w.next == null)
logger.info("\n");
previous = prediction;
}
}
use of edu.illinois.cs.cogcomp.lbjava.nlp.Word in project cogcomp-nlp by CogComp.
the class SegmentTagPlain method main.
public static void main(String[] args) {
String taggerName = null;
String inputFile = null;
String parserName = null;
try {
taggerName = args[0];
inputFile = args[1];
if (args.length > 2) {
parserName = args[2];
if (args.length > 3)
throw new Exception();
}
} catch (Exception e) {
System.err.println("usage: java edu.illinois.cs.cogcomp.lbjava.edu.illinois.cs.cogcomp.lbjava.nlp.seg.SegmentTagPlain <word classifier> " + "<input file> \\\n" + " [<parser>]");
System.exit(1);
}
Classifier tagger = ClassUtils.getClassifier(taggerName);
Parser parser;
if (parserName == null)
parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(inputFile)));
else
parser = ClassUtils.getParser(parserName, new Class[] { Parser.class }, new Parser[] { new WordSplitter(new SentenceSplitter(inputFile)) });
String previous = "";
for (Word w = (Word) parser.next(); w != null; w = (Word) parser.next()) {
String prediction = tagger.discreteValue(w);
if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
System.out.print("[" + prediction.substring(2) + " ");
System.out.print(w.form + " ");
if (!prediction.equals("O") && (w.next == null || tagger.discreteValue(w.next).equals("O") || tagger.discreteValue(w.next).startsWith("B-") || !tagger.discreteValue(w.next).endsWith(prediction.substring(2))))
System.out.print("] ");
if (w.next == null)
System.out.println();
previous = prediction;
}
}
Aggregations