Examples with AgreementResult de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult used on opensource projects

Example 16 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method twoEmptyCasTest.

@Test
public void twoEmptyCasTest() throws Exception {
    String text = "";
    JCas user1Cas = JCasFactory.createJCas();
    user1Cas.setDocumentText(text);
    JCas user2Cas = JCasFactory.createJCas();
    user2Cas.setDocumentText(text);
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(user1Cas));
    casByUser.put("user2", asList(user2Cas));
    List<String> entryTypes = asList(Lemma.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(new SpanDiffAdapter(Lemma.class.getName()));
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(0, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "value", casByUser);
    assertEquals(Double.NaN, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 17 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method twoWithoutLabelTest.

@Test
public void twoWithoutLabelTest() throws Exception {
    JCas user1 = JCasFactory.createJCas();
    user1.setDocumentText("test");
    new POS(user1, 0, 1).addToIndexes();
    new POS(user1, 1, 2).addToIndexes();
    POS p1 = new POS(user1, 3, 4);
    p1.setPosValue("A");
    p1.addToIndexes();
    JCas user2 = JCasFactory.createJCas();
    user2.setDocumentText("test");
    new POS(user2, 0, 1).addToIndexes();
    new POS(user2, 2, 3).addToIndexes();
    POS p2 = new POS(user2, 3, 4);
    p2.setPosValue("B");
    p2.addToIndexes();
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(user1));
    casByUser.put("user2", asList(user2));
    List<String> entryTypes = asList(POS.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    AgreementResult agreement = AgreementUtils.getAgreement(ConcreteAgreementMeasure.KRIPPENDORFF_ALPHA_NOMINAL_AGREEMENT, false, result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(4, agreement.getTotalSetCount());
    assertEquals(0, agreement.getIrrelevantSets().size());
    // the following two counts are zero because the incomplete sets are not excluded!
    assertEquals(2, agreement.getIncompleteSetsByPosition().size());
    assertEquals(0, agreement.getIncompleteSetsByLabel().size());
    assertEquals(3, agreement.getSetsWithDifferences().size());
    assertEquals(4, agreement.getRelevantSetCount());
    assertEquals(0.4, agreement.getAgreement(), 0.01);
    ICodingAnnotationItem item1 = agreement.getStudy().getItem(0);
    ICodingAnnotationItem item2 = agreement.getStudy().getItem(1);
    ICodingAnnotationItem item3 = agreement.getStudy().getItem(2);
    assertEquals("", item1.getUnit(0).getCategory());
    assertEquals("", item1.getUnit(1).getCategory());
    assertEquals("", item2.getUnit(0).getCategory());
    assertEquals(null, item2.getUnit(1).getCategory());
    assertEquals(null, item3.getUnit(0).getCategory());
    assertEquals("", item3.getUnit(1).getCategory());
}

Example 18 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method relationDistanceTest.

@Test
public void relationDistanceTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/relationDistance/user1.conll", "casdiff/relationDistance/user2.conll");
    List<String> entryTypes = asList(Dependency.class.getName());
    List<? extends DiffAdapter> diffAdapters = asList(new ArcDiffAdapter(Dependency.class.getName(), "Dependent", "Governor", "DependencyType"));
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(27, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(2, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "DependencyType", casByUser);
    assertEquals(1.0, agreement.getAgreement(), 0.000001d);
    assertEquals(2, agreement.getIncompleteSetsByPosition().size());
}

Example 19 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method multipleEmptyCasWithMissingOnesTest.

@Test
public void multipleEmptyCasWithMissingOnesTest() throws Exception {
    String text = "";
    JCas user1Cas1 = null;
    JCas user1Cas2 = null;
    JCas user1Cas3 = JCasFactory.createJCas();
    user1Cas3.setDocumentText(text);
    JCas user1Cas4 = JCasFactory.createJCas();
    user1Cas4.setDocumentText(text);
    JCas user2Cas1 = JCasFactory.createJCas();
    user2Cas1.setDocumentText(text);
    JCas user2Cas2 = null;
    JCas user2Cas3 = null;
    JCas user2Cas4 = JCasFactory.createJCas();
    user2Cas4.setDocumentText(text);
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(user1Cas1, user1Cas2, user1Cas3, user1Cas4));
    casByUser.put("user2", asList(user2Cas1, user2Cas2, user2Cas3, user2Cas4));
    List<String> entryTypes = asList(Lemma.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(new SpanDiffAdapter(Lemma.class.getName()));
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(0, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "value", casByUser);
    assertEquals(Double.NaN, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 20 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method singleDifferencesTest.

@Test
public void singleDifferencesTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/singleSpanDifference/user1.conll", "casdiff/singleSpanDifference/user2.conll");
    List<String> entryTypes = asList(POS.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(1, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(0.0d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}