Examples with CasDiff de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff used on opensource projects

Example 1 with CasDiff

use of de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff in project webanno by webanno.

the class PairwiseCodingAgreementTable method exportAllAgreements.

private IResourceStream exportAllAgreements() {
    return new AbstractResourceStream() {

        private static final long serialVersionUID = 1L;

        @Override
        public InputStream getInputStream() throws ResourceStreamNotFoundException {
            AnnotationFeature feature = getModelObject().getFeature();
            DefaultAgreementTraits traits = getModelObject().getTraits();
            Map<String, List<CAS>> casMap = casMapSupplier.get();
            List<DiffAdapter> adapters = CasDiff.getDiffAdapters(annotationService, asList(feature.getLayer()));
            CasDiff diff = doDiff(adapters, traits.getLinkCompareBehavior(), casMap);
            // AgreementResult agreementResult = AgreementUtils.makeStudy(diff,
            // feature.getLayer().getName(), feature.getName(),
            // pref.excludeIncomplete, casMap);
            // TODO: for the moment, we always include incomplete annotations during this
            // export.
            CodingAgreementResult agreementResult = makeCodingStudy(diff, feature.getLayer().getName(), feature.getName(), false, casMap);
            try {
                return AgreementUtils.generateCsvReport(agreementResult);
            } catch (Exception e) {
                // FIXME Is there some better error handling here?
                LOG.error("Unable to generate report", e);
                throw new ResourceStreamNotFoundException(e);
            }
        }

        @Override
        public void close() throws IOException {
        // Nothing to do
        }
    };
}

Example 2 with CasDiff

use of de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffSpanAnnotation.

@Test
public void testTwoUserDiffSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaspandiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(POS_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 3 with CasDiff

use of de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffArcAndSpanAnnotation.

@Test
public void testTwoUserDiffArcAndSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaspanarcdiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(DEPENDENCY_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    CodingAgreementResult agreement = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    AgreementUtils.dumpAgreementStudy(System.out, agreement);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 4 with CasDiff

use of de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffArcAnnotation.

@Test
public void testTwoUserDiffArcAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappaarcdiff);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(DEPENDENCY_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}

Example 5 with CasDiff

use of de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserSameAnnotation.

@Test
public void testTwoUserSameAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, CAS> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas);
    userCases.put(user2, kappatestCas);
    Map<SourceDocument, Map<User, CAS>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    CasDiff diff = doDiff(asList(POS_DIFF_ADAPTER), LINK_TARGET_AS_LABEL, convert(userCases));
    DiffResult result = diff.toResult();
    AgreementResult agreement = getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    result.print(System.out);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001);
    assertEquals(9, result.size());
    assertEquals(0, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
}