Examples with AgreementResult de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult used on opensource projects

Example 11 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserSameAnnotation.

@Test
public void testTwoUserSameAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, JCas> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas.getJCas());
    userCases.put(user2, kappatestCas.getJCas());
    Map<SourceDocument, Map<User, JCas>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    DiffResult diff = CasDiff2.doDiff(POS.class, new SpanDiffAdapter(POS.class, "PosValue"), LinkCompareBehavior.LINK_TARGET_AS_LABEL, convert(userCases));
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    diff.print(System.out);
    assertEquals(1.0d, agreement.getAgreement(), 0.000001);
    assertEquals(9, diff.size());
    assertEquals(0, diff.getDifferingConfigurationSets().size());
    assertEquals(0, diff.getIncompleteConfigurationSets().size());
}

Example 12 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffSpanAnnotation.

@Test
public void testTwoUserDiffSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, JCas> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas.getJCas());
    userCases.put(user2, kappaspandiff.getJCas());
    Map<SourceDocument, Map<User, JCas>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    DiffResult diff = CasDiff2.doDiff(POS.class, new SpanDiffAdapter(POS.class, "PosValue"), LinkCompareBehavior.LINK_TARGET_AS_LABEL, convert(userCases));
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, POS.class.getName(), "PosValue", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    diff.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, diff.size());
    assertEquals(1, diff.getDifferingConfigurationSets().size());
    assertEquals(0, diff.getIncompleteConfigurationSets().size());
}

Example 13 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class AgreementTable method makeDownloadBehavior.

private Behavior makeDownloadBehavior(final String aKey1, final String aKey2) {
    return new AjaxEventBehavior("click") {

        private static final long serialVersionUID = 1L;

        @Override
        protected void onEvent(AjaxRequestTarget aTarget) {
            AJAXDownload download = new AJAXDownload() {

                private static final long serialVersionUID = 1L;

                @Override
                protected IResourceStream getResourceStream() {
                    return new AbstractResourceStream() {

                        private static final long serialVersionUID = 1L;

                        @Override
                        public InputStream getInputStream() throws ResourceStreamNotFoundException {
                            try {
                                AgreementResult result = AgreementTable.this.getModelObject().getStudy(aKey1, aKey2);
                                switch(settings.getObject().exportFormat) {
                                    case CSV:
                                        return AgreementUtils.generateCsvReport(result);
                                    case DEBUG:
                                        return generateDebugReport(result);
                                    default:
                                        throw new IllegalStateException("Unknown export format [" + settings.getObject().exportFormat + "]");
                                }
                            } catch (Exception e) {
                                // FIXME Is there some better error handling here?
                                LOG.error("Unable to generate agreement report", e);
                                throw new ResourceStreamNotFoundException(e);
                            }
                        }

                        @Override
                        public void close() throws IOException {
                        // Nothing to do
                        }
                    };
                }
            };
            getComponent().add(download);
            download.initiate(aTarget, "agreement" + settings.getObject().exportFormat.getExtension());
        }
    };
}

Example 14 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testThreeUserDiffArcAndSpanAnnotation.

@Test
public void testThreeUserDiffArcAndSpanAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    userDocs.put(user3, asList(document));
    Map<User, JCas> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas.getJCas());
    userCases.put(user2, kappaspandiff.getJCas());
    userCases.put(user3, kappaspanarcdiff.getJCas());
    Map<SourceDocument, Map<User, JCas>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    DiffResult diff = CasDiff2.doDiff(asList(POS.class.getName(), Dependency.class.getName()), asList(SpanDiffAdapter.POS, ArcDiffAdapter.DEPENDENCY), LinkCompareBehavior.LINK_TARGET_AS_LABEL, convert(userCases));
    Map<String, List<JCas>> user1and2 = convert(userCases);
    user1and2.remove("user3");
    AgreementResult agreement12 = AgreementUtils.getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user1and2);
    Map<String, List<JCas>> user2and3 = convert(userCases);
    user2and3.remove("user1");
    AgreementResult agreement23 = AgreementUtils.getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user2and3);
    Map<String, List<JCas>> user1and3 = convert(userCases);
    user1and3.remove("user2");
    AgreementResult agreement13 = AgreementUtils.getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", user1and3);
    // Asserts
    diff.print(System.out);
    System.out.printf("New agreement 1/2: %s%n", agreement12.toString());
    System.out.printf("New agreement 2/3: %s%n", agreement23.toString());
    System.out.printf("New agreement 1/3: %s%n", agreement13.toString());
}

Example 15 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class TwoPairedKappaTest method testTwoUserDiffArcAnnotation.

@Test
public void testTwoUserDiffArcAnnotation() throws Exception {
    Map<User, List<SourceDocument>> userDocs = new HashMap<>();
    userDocs.put(user1, asList(document));
    userDocs.put(user2, asList(document));
    Map<User, JCas> userCases = new HashMap<>();
    userCases.put(user1, kappatestCas.getJCas());
    userCases.put(user2, kappaarcdiff.getJCas());
    Map<SourceDocument, Map<User, JCas>> documentJCases = new HashMap<>();
    documentJCases.put(document, userCases);
    // Check against new impl
    DiffResult diff = CasDiff2.doDiff(Dependency.class, new ArcDiffAdapter(Dependency.class, "Dependent", "Governor", "DependencyType"), LinkCompareBehavior.LINK_TARGET_AS_LABEL, convert(userCases));
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, Dependency.class.getName(), "DependencyType", convert(userCases));
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    diff.print(System.out);
    assertEquals(0.86153d, agreement.getAgreement(), 0.00001d);
    assertEquals(9, diff.size());
    assertEquals(1, diff.getDifferingConfigurationSets().size());
    assertEquals(0, diff.getIncompleteConfigurationSets().size());
}