Examples with WordSplitter edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter used on opensource projects

Example 1 with WordSplitter

use of edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter in project cogcomp-nlp by CogComp.

the class TestDiff method testDiff.

@Test
public void testDiff() {
    POSTagger tagger = new POSTagger();
    Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(testFile)));
    String sentence = "";
    int sentenceCounter = 0;
    int tokenCounter = 0;
    int correctCounter = 0;
    for (Token word = (Token) parser.next(); word != null; word = (Token) parser.next()) {
        String tag = tagger.discreteValue(word);
        if (refTags.get(tokenCounter).equals(tag)) {
            correctCounter++;
        }
        tokenCounter++;
    }
    double result = ((double) correctCounter) / tokenCounter;
    if (result < thresholdAcc) {
        fail("Tagger performance is insufficient: " + "\nProduced: " + result + "\nExpected: " + thresholdAcc);
    }
}

Example 2 with WordSplitter

use of edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter in project cogcomp-nlp by CogComp.

the class NEWord method splitWord.

/*
     * Used for some tokenization schemes.
     */
private static Vector<NEWord> splitWord(NEWord word) {
    String[] sentence = { word.form + " " };
    Parser parser = new WordSplitter(new SentenceSplitter(sentence));
    LinkedVector words = (LinkedVector) parser.next();
    Vector<NEWord> res = new Vector<>();
    if (words == null) {
        res.add(word);
        return res;
    }
    String label = word.neLabel;
    for (int i = 0; i < words.size(); i++) {
        if (label.contains("B-") && i > 0)
            label = "I-" + label.substring(2);
        NEWord w = new NEWord(new Word(((Word) words.get(i)).form), null, label);
        res.addElement(w);
    }
    return res;
}

Example 3 with WordSplitter

use of edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter in project cogcomp-nlp by CogComp.

the class TestDiff method testDiff.

@Test
public void testDiff() {
    Chunker tagger = new Chunker();
    Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(testFile)));
    String previous = "";
    String sentence = "";
    int sentenceCounter = 0;
    for (Token w = (Token) parser.next(); w != null; w = (Token) parser.next()) {
        String prediction = tagger.discreteValue(w);
        if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
            sentence += ("[" + prediction.substring(2) + " ");
        sentence += ("(" + w.partOfSpeech + " " + w.form + ") ");
        if (!prediction.equals("O") && (w.next == null || tagger.discreteValue(w.next).equals("O") || tagger.discreteValue(w.next).startsWith("B-") || !tagger.discreteValue(w.next).endsWith(prediction.substring(2))))
            sentence += ("] ");
        if (w.next == null) {
            sentence = sentence.trim();
            String refSentence = refSentences.get(sentenceCounter).trim();
            if (!sentence.equals(refSentence))
                fail("Produced output doesn't match reference: " + "\nProduced: " + sentence + "\nExpected: " + refSentence);
            sentence = "";
            sentenceCounter++;
        }
        previous = prediction;
    }
}

Example 4 with WordSplitter

use of edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter in project cogcomp-nlp by CogComp.

the class ChunksAndPOSTags method main.

public static void main(String[] args) {
    String filename = null;
    try {
        filename = args[0];
        if (args.length > 1)
            throw new Exception();
    } catch (Exception e) {
        System.err.println("usage: java edu.illinois.cs.cogcomp.chunker.main.ChunksAndPOSTags <input file>");
        System.exit(1);
    }
    Chunker chunker = new Chunker();
    Parser parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(filename)));
    String previous = "";
    for (Word w = (Word) parser.next(); w != null; w = (Word) parser.next()) {
        String prediction = chunker.discreteValue(w);
        if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
            logger.info("[" + prediction.substring(2) + " ");
        logger.info("(" + w.partOfSpeech + " " + w.form + ") ");
        if (!prediction.equals("O") && (w.next == null || chunker.discreteValue(w.next).equals("O") || chunker.discreteValue(w.next).startsWith("B-") || !chunker.discreteValue(w.next).endsWith(prediction.substring(2))))
            logger.info("] ");
        if (w.next == null)
            logger.info("\n");
        previous = prediction;
    }
}

Example 5 with WordSplitter

use of edu.illinois.cs.cogcomp.lbjava.nlp.WordSplitter in project cogcomp-nlp by CogComp.

the class SegmentTagPlain method main.

public static void main(String[] args) {
    String taggerName = null;
    String inputFile = null;
    String parserName = null;
    try {
        taggerName = args[0];
        inputFile = args[1];
        if (args.length > 2) {
            parserName = args[2];
            if (args.length > 3)
                throw new Exception();
        }
    } catch (Exception e) {
        System.err.println("usage: java edu.illinois.cs.cogcomp.lbjava.edu.illinois.cs.cogcomp.lbjava.nlp.seg.SegmentTagPlain <word classifier> " + "<input file> \\\n" + "                                         [<parser>]");
        System.exit(1);
    }
    Classifier tagger = ClassUtils.getClassifier(taggerName);
    Parser parser;
    if (parserName == null)
        parser = new PlainToTokenParser(new WordSplitter(new SentenceSplitter(inputFile)));
    else
        parser = ClassUtils.getParser(parserName, new Class[] { Parser.class }, new Parser[] { new WordSplitter(new SentenceSplitter(inputFile)) });
    String previous = "";
    for (Word w = (Word) parser.next(); w != null; w = (Word) parser.next()) {
        String prediction = tagger.discreteValue(w);
        if (prediction.startsWith("B-") || prediction.startsWith("I-") && !previous.endsWith(prediction.substring(2)))
            System.out.print("[" + prediction.substring(2) + " ");
        System.out.print(w.form + " ");
        if (!prediction.equals("O") && (w.next == null || tagger.discreteValue(w.next).equals("O") || tagger.discreteValue(w.next).startsWith("B-") || !tagger.discreteValue(w.next).endsWith(prediction.substring(2))))
            System.out.print("] ");
        if (w.next == null)
            System.out.println();
        previous = prediction;
    }
}