Examples with ICodingAnnotationItem org.dkpro.statistics.agreement.coding.ICodingAnnotationItem used on opensource projects

Example 1 with ICodingAnnotationItem

use of org.dkpro.statistics.agreement.coding.ICodingAnnotationItem in project webanno by webanno.

the class FleissKappaAgreementMeasureTest method twoWithoutLabelTest.

@Test
public void twoWithoutLabelTest() throws Exception {
    PairwiseAnnotationResult<CodingAgreementResult> agreement = twoWithoutLabelTest(sut, traits);
    CodingAgreementResult result = agreement.getStudy("user1", "user2");
    ICodingAnnotationItem item1 = result.getStudy().getItem(0);
    ICodingAnnotationItem item2 = result.getStudy().getItem(1);
    assertEquals("", item1.getUnit(0).getCategory());
    assertEquals("", item1.getUnit(1).getCategory());
    assertEquals("A", item2.getUnit(0).getCategory());
    assertEquals(4, result.getTotalSetCount());
    assertEquals(0, result.getIrrelevantSets().size());
    assertEquals(2, result.getIncompleteSetsByPosition().size());
    assertEquals(0, result.getIncompleteSetsByLabel().size());
    assertEquals(1, result.getSetsWithDifferences().size());
    assertEquals(4, result.getRelevantSetCount());
    assertEquals(0.2, result.getAgreement(), 0.01);
}

Example 2 with ICodingAnnotationItem

use of org.dkpro.statistics.agreement.coding.ICodingAnnotationItem in project webanno by webanno.

the class KrippendorffAlphaNominalAgreementMeasureTest method testTwoWithoutLabel_noExcludeIncomplete.

@Test
public void testTwoWithoutLabel_noExcludeIncomplete() throws Exception {
    traits.setExcludeIncomplete(false);
    PairwiseAnnotationResult<CodingAgreementResult> agreement = twoWithoutLabelTest(sut, traits);
    CodingAgreementResult result = agreement.getStudy("user1", "user2");
    ICodingAnnotationItem item1 = result.getStudy().getItem(0);
    ICodingAnnotationItem item2 = result.getStudy().getItem(1);
    ICodingAnnotationItem item3 = result.getStudy().getItem(2);
    assertEquals("", item1.getUnit(0).getCategory());
    assertEquals("", item1.getUnit(1).getCategory());
    assertEquals("", item2.getUnit(0).getCategory());
    assertEquals(null, item2.getUnit(1).getCategory());
    assertEquals(null, item3.getUnit(0).getCategory());
    assertEquals("", item3.getUnit(1).getCategory());
    assertEquals(4, result.getTotalSetCount());
    assertEquals(0, result.getIrrelevantSets().size());
    // the following two counts are zero because the incomplete sets are not excluded!
    assertEquals(2, result.getIncompleteSetsByPosition().size());
    assertEquals(0, result.getIncompleteSetsByLabel().size());
    assertEquals(3, result.getSetsWithDifferences().size());
    assertEquals(4, result.getRelevantSetCount());
    assertEquals(0.4, result.getAgreement(), 0.01);
}

Example 3 with ICodingAnnotationItem

use of org.dkpro.statistics.agreement.coding.ICodingAnnotationItem in project webanno by webanno.

the class CohenKappaAgreementMeasureTest method twoWithoutLabelTest.

@Test
public void twoWithoutLabelTest() throws Exception {
    PairwiseAnnotationResult<CodingAgreementResult> agreement = twoWithoutLabelTest(sut, traits);
    CodingAgreementResult result = agreement.getStudy("user1", "user2");
    result.getDiff().print(System.out);
    ICodingAnnotationItem item1 = result.getStudy().getItem(0);
    ICodingAnnotationItem item2 = result.getStudy().getItem(1);
    assertEquals("", item1.getUnit(0).getCategory());
    assertEquals("", item1.getUnit(1).getCategory());
    assertEquals("A", item2.getUnit(0).getCategory());
    assertEquals(4, result.getTotalSetCount());
    assertEquals(0, result.getIrrelevantSets().size());
    assertEquals(2, result.getIncompleteSetsByPosition().size());
    assertEquals(0, result.getIncompleteSetsByLabel().size());
    assertEquals(1, result.getSetsWithDifferences().size());
    assertEquals(4, result.getRelevantSetCount());
    assertEquals(0.333, result.getAgreement(), 0.01);
}

Example 4 with ICodingAnnotationItem

use of org.dkpro.statistics.agreement.coding.ICodingAnnotationItem in project webanno by webanno.

the class AgreementUtils method configurationSetsWithItemsToCsv.

private static void configurationSetsWithItemsToCsv(CSVPrinter aOut, AgreementResult<ICodingAnnotationStudy> aAgreement, List<ConfigurationSet> aSets) throws IOException {
    List<String> headers = new ArrayList<>(asList("Type", "Collection", "Document", "Layer", "Feature", "Position"));
    headers.addAll(aAgreement.getCasGroupIds());
    aOut.printRecord(headers);
    int i = 0;
    for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
        Position pos = aSets.get(i).getPosition();
        List<String> values = new ArrayList<>();
        values.add(pos.getClass().getSimpleName());
        values.add(pos.getCollectionId());
        values.add(pos.getDocumentId());
        values.add(pos.getType());
        values.add(aAgreement.getFeature());
        values.add(aSets.get(i).getPosition().toMinimalString());
        for (IAnnotationUnit unit : item.getUnits()) {
            values.add(String.valueOf(unit.getCategory()));
        }
        aOut.printRecord(values);
        i++;
    }
}

Example 5 with ICodingAnnotationItem

use of org.dkpro.statistics.agreement.coding.ICodingAnnotationItem in project webanno by webanno.

the class AgreementUtils method dumpAgreementConfigurationSetsWithItems.

private static void dumpAgreementConfigurationSetsWithItems(PrintStream aOut, AgreementResult<ICodingAnnotationStudy> aAgreement, List<ConfigurationSet> aSets) {
    int i = 0;
    for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
        StringBuilder sb = new StringBuilder();
        sb.append(aSets.get(i).getPosition());
        for (IAnnotationUnit unit : item.getUnits()) {
            if (sb.length() > 0) {
                sb.append(" \t");
            }
            sb.append(unit.getCategory());
        }
        aOut.println(sb);
        i++;
    }
}