Examples with AgreementResult de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult used on opensource projects

Example 21 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method relationStackedSpansTest.

@Test
public void relationStackedSpansTest() throws Exception {
    TypeSystemDescription global = TypeSystemDescriptionFactory.createTypeSystemDescription();
    TypeSystemDescription local = TypeSystemDescriptionFactory.createTypeSystemDescriptionFromPath("src/test/resources/desc/type/webannoTestTypes.xml");
    TypeSystemDescription merged = CasCreationUtils.mergeTypeSystems(asList(global, local));
    TokenBuilder<Token, Sentence> tb = new TokenBuilder<>(Token.class, Sentence.class);
    JCas jcasA = JCasFactory.createJCas(merged);
    {
        CAS casA = jcasA.getCas();
        tb.buildTokens(jcasA, "This is a test .");
        List<Token> tokensA = new ArrayList<>(select(jcasA, Token.class));
        Token t1A = tokensA.get(0);
        Token t2A = tokensA.get(tokensA.size() - 1);
        NamedEntity govA = new NamedEntity(jcasA, t1A.getBegin(), t1A.getEnd());
        govA.addToIndexes();
        // Here we add a stacked named entity!
        new NamedEntity(jcasA, t1A.getBegin(), t1A.getEnd()).addToIndexes();
        NamedEntity depA = new NamedEntity(jcasA, t2A.getBegin(), t2A.getEnd());
        depA.addToIndexes();
        Type relationTypeA = casA.getTypeSystem().getType("webanno.custom.Relation");
        AnnotationFS fs1A = casA.createAnnotation(relationTypeA, depA.getBegin(), depA.getEnd());
        FSUtil.setFeature(fs1A, "Governor", govA);
        FSUtil.setFeature(fs1A, "Dependent", depA);
        FSUtil.setFeature(fs1A, "value", "REL");
        casA.addFsToIndexes(fs1A);
    }
    JCas jcasB = JCasFactory.createJCas(merged);
    {
        CAS casB = jcasB.getCas();
        tb.buildTokens(jcasB, "This is a test .");
        List<Token> tokensB = new ArrayList<>(select(jcasB, Token.class));
        Token t1B = tokensB.get(0);
        Token t2B = tokensB.get(tokensB.size() - 1);
        NamedEntity govB = new NamedEntity(jcasB, t1B.getBegin(), t1B.getEnd());
        govB.addToIndexes();
        NamedEntity depB = new NamedEntity(jcasB, t2B.getBegin(), t2B.getEnd());
        depB.addToIndexes();
        Type relationTypeB = casB.getTypeSystem().getType("webanno.custom.Relation");
        AnnotationFS fs1B = casB.createAnnotation(relationTypeB, depB.getBegin(), depB.getEnd());
        FSUtil.setFeature(fs1B, "Governor", govB);
        FSUtil.setFeature(fs1B, "Dependent", depB);
        FSUtil.setFeature(fs1B, "value", "REL");
        casB.addFsToIndexes(fs1B);
    }
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(jcasA));
    casByUser.put("user2", asList(jcasB));
    List<String> entryTypes = asList("webanno.custom.Relation");
    List<? extends DiffAdapter> diffAdapters = asList(new ArcDiffAdapter("webanno.custom.Relation", WebAnnoConst.FEAT_REL_TARGET, WebAnnoConst.FEAT_REL_SOURCE, "value"));
    DiffResult diff = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    diff.print(System.out);
    assertEquals(1, diff.size());
    assertEquals(0, diff.getDifferingConfigurationSets().size());
    assertEquals(0, diff.getIncompleteConfigurationSets().size());
    // Check against new impl
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, "webanno.custom.Relation", "value", casByUser);
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    AgreementUtils.dumpAgreementStudy(System.out, agreement);
    assertEquals(1, agreement.getPluralitySets().size());
}

Example 22 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method multiLinkWithRoleLabelDifferenceTest.

@Test
public void multiLinkWithRoleLabelDifferenceTest() throws Exception {
    JCas jcasA = JCasFactory.createJCas(DiffUtils.createMultiLinkWithRoleTestTypeSytem());
    DiffUtils.makeLinkHostFS(jcasA, 0, 0, DiffUtils.makeLinkFS(jcasA, "slot1", 0, 0));
    JCas jcasB = JCasFactory.createJCas(DiffUtils.createMultiLinkWithRoleTestTypeSytem());
    DiffUtils.makeLinkHostFS(jcasB, 0, 0, DiffUtils.makeLinkFS(jcasB, "slot2", 0, 0));
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(jcasA));
    casByUser.put("user2", asList(jcasB));
    List<String> entryTypes = asList(DiffUtils.HOST_TYPE);
    SpanDiffAdapter adapter = new SpanDiffAdapter(DiffUtils.HOST_TYPE);
    adapter.addLinkFeature("links", "role", "target");
    List<? extends DiffAdapter> diffAdapters = asList(adapter);
    DiffResult diff = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    diff.print(System.out);
    assertEquals(3, diff.size());
    assertEquals(0, diff.getDifferingConfigurationSets().size());
    assertEquals(2, diff.getIncompleteConfigurationSets().size());
    // Check against new impl
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, DiffUtils.HOST_TYPE, "links", casByUser);
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    AgreementUtils.dumpAgreementStudy(System.out, agreement);
    assertEquals(Double.NaN, agreement.getAgreement(), 0.00001d);
}

Example 23 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method singleNoDifferencesWithAdditionalCas2Test.

@Test
public void singleNoDifferencesWithAdditionalCas2Test() throws Exception {
    JCas user1 = JCasFactory.createJCas();
    user1.setDocumentText("test");
    JCas user2 = JCasFactory.createJCas();
    user2.setDocumentText("test");
    JCas user3 = JCasFactory.createJCas();
    user3.setDocumentText("test");
    POS pos3 = new POS(user3, 0, 4);
    pos3.addToIndexes();
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(user1));
    casByUser.put("user2", asList(user2));
    casByUser.put("user3", asList(user3));
    List<String> entryTypes = asList(POS.class.getName());
    List<SpanDiffAdapter> diffAdapters = asList(SpanDiffAdapter.POS);
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    casByUser.remove("user3");
    AgreementResult agreement = AgreementUtils.getAgreement(ConcreteAgreementMeasure.KRIPPENDORFF_ALPHA_NOMINAL_AGREEMENT, false, result, entryTypes.get(0), "PosValue", casByUser);
    assertEquals(1, agreement.getTotalSetCount());
    assertEquals(1, agreement.getIrrelevantSets().size());
    assertEquals(0, agreement.getRelevantSetCount());
}

Example 24 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method multiLinkWithRoleTargetDifferenceTest.

@Test
public void multiLinkWithRoleTargetDifferenceTest() throws Exception {
    JCas jcasA = JCasFactory.createJCas(DiffUtils.createMultiLinkWithRoleTestTypeSytem());
    DiffUtils.makeLinkHostFS(jcasA, 0, 0, DiffUtils.makeLinkFS(jcasA, "slot1", 0, 0));
    JCas jcasB = JCasFactory.createJCas(DiffUtils.createMultiLinkWithRoleTestTypeSytem());
    DiffUtils.makeLinkHostFS(jcasB, 0, 0, DiffUtils.makeLinkFS(jcasB, "slot1", 10, 10));
    Map<String, List<JCas>> casByUser = new LinkedHashMap<>();
    casByUser.put("user1", asList(jcasA));
    casByUser.put("user2", asList(jcasB));
    List<String> entryTypes = asList(DiffUtils.HOST_TYPE);
    SpanDiffAdapter adapter = new SpanDiffAdapter(DiffUtils.HOST_TYPE);
    adapter.addLinkFeature("links", "role", "target");
    List<? extends DiffAdapter> diffAdapters = asList(adapter);
    DiffResult diff = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    diff.print(System.out);
    assertEquals(2, diff.size());
    assertEquals(1, diff.getDifferingConfigurationSets().size());
    assertEquals(0, diff.getIncompleteConfigurationSets().size());
    // Check against new impl
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(diff, DiffUtils.HOST_TYPE, "links", casByUser);
    // Asserts
    System.out.printf("Agreement: %s%n", agreement.toString());
    AgreementUtils.dumpAgreementStudy(System.out, agreement);
    assertEquals(0.0, agreement.getAgreement(), 0.00001d);
}

Example 25 with AgreementResult

use of de.tudarmstadt.ukp.clarin.webanno.curation.agreement.AgreementUtils.AgreementResult in project webanno by webanno.

the class CasDiff2Test method relationLabelTest.

@Test
public void relationLabelTest() throws Exception {
    Map<String, List<JCas>> casByUser = DiffUtils.load("casdiff/relationLabel/user1.conll", "casdiff/relationLabel/user2.conll");
    List<String> entryTypes = asList(Dependency.class.getName());
    List<? extends DiffAdapter> diffAdapters = asList(new ArcDiffAdapter(Dependency.class.getName(), "Dependent", "Governor", "DependencyType"));
    DiffResult result = CasDiff2.doDiff(entryTypes, diffAdapters, LinkCompareBehavior.LINK_TARGET_AS_LABEL, casByUser);
    result.print(System.out);
    assertEquals(26, result.size());
    assertEquals(1, result.getDifferingConfigurationSets().size());
    assertEquals(0, result.getIncompleteConfigurationSets().size());
    AgreementResult agreement = AgreementUtils.getCohenKappaAgreement(result, entryTypes.get(0), "DependencyType", casByUser);
    assertEquals(0.958199d, agreement.getAgreement(), 0.000001d);
    assertEquals(0, agreement.getIncompleteSetsByPosition().size());
}