Examples with TokenLabelView edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView used on opensource projects

Example 16 with TokenLabelView

use of edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView in project cogcomp-nlp by CogComp.

the class CommaTest method setUp.

@Override
public void setUp() throws Exception {
    String[] tokens = "Says Gayle Key , a mathematics teacher , `` Hello world . ''".split("\\s+");
    TextAnnotation ta = BasicTextAnnotationBuilder.createTextAnnotationFromTokens(Collections.singletonList(tokens));
    TokenLabelView tlv = new TokenLabelView(ViewNames.POS, "Test", ta, 1.0);
    tlv.addTokenLabel(0, "VBZ", 1d);
    tlv.addTokenLabel(1, "NNP", 1d);
    tlv.addTokenLabel(2, "NNP", 1d);
    tlv.addTokenLabel(3, ",", 1d);
    tlv.addTokenLabel(4, "DT", 1d);
    tlv.addTokenLabel(5, "NN", 1d);
    tlv.addTokenLabel(6, "NN", 1d);
    tlv.addTokenLabel(7, ",", 1d);
    tlv.addTokenLabel(8, "``", 1d);
    tlv.addTokenLabel(9, "UH", 1d);
    tlv.addTokenLabel(10, "NN", 1d);
    tlv.addTokenLabel(11, ".", 1d);
    tlv.addTokenLabel(12, "''", 1d);
    TreeView parse = new TreeView(ViewNames.PARSE_STANFORD, "Test", ta, 1.0);
    String treeString = "(ROOT" + "  (SINV" + "    (VP (VBZ Says))" + "    (NP (NNP Gayle) (NNP Key))" + "    (, ,)" + "    (S" + "      (NP (DT a) (NNS mathematics))" + "      (VP (VBZ teacher) (, ,) (`` ``)" + "        (NP" + "          (INTJ (UH Hello))" + "          (NP (NN world)))))" + "    (. .) ('' '')))";
    parse.setParseTree(0, TreeParserFactory.getStringTreeParser().parse(treeString));
    SpanLabelView ner = new SpanLabelView(ViewNames.NER_CONLL, "Test", ta, 1.0);
    ner.addSpanLabel(1, 3, "PER", 1.0);
    SpanLabelView shallowParse = new SpanLabelView(ViewNames.SHALLOW_PARSE, "Test", ta, 1.0);
    shallowParse.addSpanLabel(0, 3, "NP", 1.0);
    shallowParse.addSpanLabel(4, 7, "NP", 1.0);
    shallowParse.addSpanLabel(9, 11, "NP", 1.0);
    // TODO dependency parse
    // TODO SRL view
    ta.addView(tlv.getViewName(), tlv);
    ta.addView(parse.getViewName(), parse);
    ta.addView(ner.getViewName(), ner);
    ta.addView(shallowParse.getViewName(), shallowParse);
    List<String> firstCommasRefinedLabels = Collections.singletonList("Substitute");
    List<String> secondCommasRefinedLabels = Arrays.asList("Substitute", "Quotation");
    CommaSRLSentence sentence = new CommaSRLSentence(ta, null, Arrays.asList(firstCommasRefinedLabels, secondCommasRefinedLabels));
    List<Comma> sentenceCommas = sentence.getCommas();
    commas = sentenceCommas.toArray(new Comma[sentenceCommas.size()]);
}

Example 17 with TokenLabelView

use of edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView in project cogcomp-nlp by CogComp.

the class VerbSensePropbankReader method addAnnotation.

private void addAnnotation(TextAnnotation ta) {
    String goldViewName = SenseManager.getGoldViewName();
    Tree<String> tree = ParseHelper.getParseTree(ViewNames.PARSE_GOLD, ta, 0);
    Tree<Pair<String, IntPair>> spanLabeledTree = ParseUtils.getSpanLabeledTree(tree);
    List<Tree<Pair<String, IntPair>>> yield = spanLabeledTree.getYield();
    TokenLabelView view = new TokenLabelView(goldViewName, "AnnotatedTreebank", ta, 1.0);
    Set<Integer> predicates = new HashSet<>();
    for (PropbankFields fields : goldFields.get(ta.getId())) {
        int start = fields.getPredicateStart(yield);
        if (predicates.contains(start))
            continue;
        predicates.add(start);
        view.addTokenLabel(start, fields.getSense(), 1.0);
        try {
            view.addTokenAttribute(start, LemmaIdentifier, fields.getLemma());
        } catch (Exception e) {
            // XXX Maybe log the exception?
            e.printStackTrace();
        }
    }
    if (view.getConstituents().size() > 0)
        ta.addView(goldViewName, view);
}

Example 18 with TokenLabelView

use of edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView in project cogcomp-nlp by CogComp.

the class VerbSenseClassifierMain method evaluate.

@CommandDescription(description = "Performs evaluation.", usage = "evaluate")
public static void evaluate() throws Exception {
    SenseManager manager = getManager(false);
    Dataset testSet = Dataset.PTBTest;
    ILPSolverFactory solver = new ILPSolverFactory(ILPSolverFactory.SolverType.JLISCuttingPlaneGurobi);
    ClassificationTester senseTester = new ClassificationTester();
    long start = System.currentTimeMillis();
    int count = 0;
    manager.getModelInfo().loadWeightVector();
    IResetableIterator<TextAnnotation> dataset = SentenceDBHandler.instance.getDataset(testSet);
    while (dataset.hasNext()) {
        TextAnnotation ta = dataset.next();
        if (!ta.hasView(SenseManager.getGoldViewName()))
            continue;
        TokenLabelView gold = (TokenLabelView) ta.getView(SenseManager.getGoldViewName());
        ILPInference inference = manager.getInference(solver, gold.getConstituents());
        assert inference != null;
        TokenLabelView prediction = inference.getOutputView();
        evaluateSense(gold, prediction, senseTester);
        count++;
        if (count % 1000 == 0) {
            long end = System.currentTimeMillis();
            log.info(count + " sentences done. Took " + (end - start) + "ms, Micro-F1 so far = " + senseTester.getMicroF1());
        }
    }
    long end = System.currentTimeMillis();
    System.out.println(count + " sentences done. Took " + (end - start) + "ms");
    System.out.println("\n\n* Sense");
    System.out.println(senseTester.getPerformanceTable(false).toOrgTable());
}

Example 19 with TokenLabelView

use of edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView in project cogcomp-nlp by CogComp.

the class VerbSenseLabeler method getPrediction.

public TokenLabelView getPrediction(TextAnnotation ta) throws Exception {
    log.debug("Input: {}", ta.getText());
    List<Constituent> predicates = manager.getPredicateDetector().getPredicates(ta);
    // If there are no verb identified, return an empty TokenLabelView
    if (predicates.isEmpty())
        return new TokenLabelView(SenseManager.getPredictedViewName(), VerbSenseConstants.systemIdentifier, ta, 1.0);
    ILPSolverFactory solver = new ILPSolverFactory(ILPSolverFactory.SolverType.JLISCuttingPlaneGurobi);
    ILPInference inference = manager.getInference(solver, predicates);
    return inference.getOutputView();
}

Example 20 with TokenLabelView

use of edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView in project cogcomp-nlp by CogComp.

the class VerbSenseLabeler method initializeDummySentenceVerb.

protected TextAnnotation initializeDummySentenceVerb() {
    List<String[]> listOfTokens = new ArrayList<>();
    listOfTokens.add(new String[] { "I", "do", "." });
    TextAnnotation ta = BasicTextAnnotationBuilder.createTextAnnotationFromTokens("", "", listOfTokens);
    TokenLabelView tlv = new TokenLabelView(ViewNames.POS, "Test", ta, 1.0);
    tlv.addTokenLabel(0, "PRP", 1d);
    tlv.addTokenLabel(1, "VBP", 1d);
    tlv.addTokenLabel(2, ".", 1d);
    ta.addView(ViewNames.POS, tlv);
    ta.addView(ViewNames.NER, new SpanLabelView(ViewNames.NER, "test", ta, 1d));
    SpanLabelView chunks = new SpanLabelView(ViewNames.SHALLOW_PARSE, "test", ta, 1d);
    chunks.addSpanLabel(0, 1, "NP", 1d);
    chunks.addSpanLabel(1, 2, "VP", 1d);
    ta.addView(ViewNames.SHALLOW_PARSE, chunks);
    TokenLabelView view = new TokenLabelView(ViewNames.LEMMA, "test", ta, 1d);
    view.addTokenLabel(0, "i", 1d);
    view.addTokenLabel(1, "do", 1d);
    view.addTokenLabel(2, ".", 1d);
    ta.addView(ViewNames.LEMMA, view);
    return ta;
}