Examples with AgreementResult de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult used on opensource projects

Example 1 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffSpanAnnotation.

@Test
public void testTwoUserDiffSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaspandiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(POS_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 2 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffArcAnnotation.

@Test
public void testTwoUserDiffArcAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaarcdiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(DEPENDENCY_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 3 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserSameAnnotation.

@Test
public void testTwoUserSameAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappatestCas);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(POS_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001);
    assertEquals(9, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 4 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testThreeUserDiffArcAndSpanAnnotation.

@Test
public void testThreeUserDiffArcAndSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    userDocs.put(user3, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaspandiff);
    userCases.put(user3, kappaspanarcdiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(POS_DIFF_ADAPTER, DEPENDENCY_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    Map<String, List<CAS>> user1and2 = convert(userCases);
    user1and2.remove("user3");
    AgreementResult agreement12 = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user1and2);
    Map<String, List<CAS>> user2and3 = convert(userCases);
    user2and3.remove("user1");
    AgreementResult agreement23 = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user2and3);
    Map<String, List<CAS>> user1and3 = convert(userCases);
    user1and3.remove("user2");
    AgreementResult agreement13 = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user1and3);
    // Asserts
    result.print(System.out);
    System.out.printf("New agreement 1/2: %s%n", agreement12.toString());
    System.out.printf("New agreement 2/3: %s%n", agreement23.toString());
    System.out.printf("New agreement 1/3: %s%n", agreement13.toString());
}