Examples with AgreementResult de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult used on opensource projects

Example 1 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method noDifferencesDependencyTest.

@Test
public void noDifferencesDependencyTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/noDifferences/data.conll", "casdiff/noDifferences/data.conll");
    List<String> entryTypes = asList(Dependency.class.getName());
    List<? extends DiffAdapter> diffAdapters = asList(ArcDiffAdapter.DEPENDENCY);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(26, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "DependencyType", casByUser);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 2 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method noDifferencesPosDependencyTest.

@Test
public void noDifferencesPosDependencyTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/noDifferences/data.conll", "casdiff/noDifferences/data.conll");
    List<String> entryTypes = asList(POS.class.getName(), Dependency.class.getName());
    List<? extends DiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS, ArcDiffAdapter.DEPENDENCY);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(52, result.size());
    assertEquals(26, result.size(POS.class.getName()));
    assertEquals(26, result.size(Dependency.class.getName()));
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 3 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method noDifferencesPosTest.

@Test
public void noDifferencesPosTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/noDifferences/data.conll", "casdiff/noDifferences/data.conll");
    List<String> entryTypes = asList(POS.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(26, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 4 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method someDifferencesTest.

@Test
public void someDifferencesTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/someDifferences/user1.conll", "casdiff/someDifferences/user2.conll");
    List<String> entryTypes = asList(POS.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(26, result.size());
    assertEquals(4, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(0.836477987d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}

Example 5 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method singleNoDifferencesTest.

@Test
public void singleNoDifferencesTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/singleSpanNoDifference/data.conll", "casdiff/singleSpanNoDifference/data.conll");
    List<String> entryTypes = asList(POS.class.getName());
    List<? extends DiffAdapter> diffAdapters = asList(new SpanDiffAdapter(POS.class.getName(), "PosValue"));
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(1, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(Double.NaN, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}