Search in sources :

Example 1 with CodingAnnotationStudy

use of org.dkpro.statistics.agreement.coding.CodingAnnotationStudy in project webanno by webanno.

the class AgreementUtils method makeCodingStudy.

private static CodingAgreementResult makeCodingStudy(CasDiff aDiff, Collection<String> aUsers, String aType, String aFeature, boolean aExcludeIncomplete, boolean aNullLabelsAsEmpty, Map<String, List<CAS>> aCasMap) {
    List<String> users = new ArrayList<>(aUsers);
    Collections.sort(users);
    List<ConfigurationSet> completeSets = new ArrayList<>();
    List<ConfigurationSet> setsWithDifferences = new ArrayList<>();
    List<ConfigurationSet> incompleteSetsByPosition = new ArrayList<>();
    List<ConfigurationSet> incompleteSetsByLabel = new ArrayList<>();
    List<ConfigurationSet> pluralitySets = new ArrayList<>();
    List<ConfigurationSet> irrelevantSets = new ArrayList<>();
    CodingAnnotationStudy study = new CodingAnnotationStudy(users.size());
    // Check if the feature we are looking at is a primitive feature or a link feature
    // We do this by looking it up in the first available CAS. Mind that at this point all
    // CASes should have exactly the same typesystem.
    CAS someCas = findSomeCas(aCasMap);
    if (someCas == null) {
        // Well... there is NOTHING here!
        // All positions are irrelevant
        aDiff.getPositions().forEach(p -> irrelevantSets.add(aDiff.getConfigurationSet(p)));
        return new CodingAgreementResult(aType, aFeature, aDiff.toResult(), study, users, completeSets, irrelevantSets, setsWithDifferences, incompleteSetsByPosition, incompleteSetsByLabel, pluralitySets, aExcludeIncomplete);
    }
    TypeSystem ts = someCas.getTypeSystem();
    // We should just do the right thing here which is: do nothing
    if (ts.getType(aType) == null) {
        // All positions are irrelevant
        aDiff.getPositions().forEach(p -> irrelevantSets.add(aDiff.getConfigurationSet(p)));
        return new CodingAgreementResult(aType, aFeature, aDiff.toResult(), study, users, completeSets, irrelevantSets, setsWithDifferences, incompleteSetsByPosition, incompleteSetsByLabel, pluralitySets, aExcludeIncomplete);
    }
    // Check that the feature really exists instead of just getting a NPE later
    if (ts.getType(aType).getFeatureByBaseName(aFeature) == null) {
        throw new IllegalArgumentException("Type [" + aType + "] has no feature called [" + aFeature + "]");
    }
    boolean isPrimitiveFeature = ts.getType(aType).getFeatureByBaseName(aFeature).getRange().isPrimitive();
    nextPosition: for (Position p : aDiff.getPositions()) {
        ConfigurationSet cfgSet = aDiff.getConfigurationSet(p);
        // Only calculate agreement for the given layer
        if (!cfgSet.getPosition().getType().equals(aType)) {
            // We don't even consider these as irrelevant, they are just filtered out
            continue;
        }
        // If the feature on a position is set, then it is a subposition
        boolean isSubPosition = p.getFeature() != null;
        // this is an inverted XOR!
        if (!(isPrimitiveFeature ^ isSubPosition)) {
            irrelevantSets.add(cfgSet);
            continue;
        }
        // feature
        if (isSubPosition && !aFeature.equals(cfgSet.getPosition().getFeature())) {
            irrelevantSets.add(cfgSet);
            continue nextPosition;
        }
        // If non of the current users has made any annotation at this position, then skip it
        if (users.stream().filter(u -> cfgSet.getCasGroupIds().contains(u)).count() == 0) {
            irrelevantSets.add(cfgSet);
            continue nextPosition;
        }
        Object[] values = new Object[users.size()];
        int i = 0;
        for (String user : users) {
            // this configuration set.
            if (!cfgSet.getCasGroupIds().contains(user)) {
                incompleteSetsByPosition.add(cfgSet);
                if (aExcludeIncomplete) {
                    // Record as incomplete
                    continue nextPosition;
                } else {
                    // Record as missing value
                    values[i] = null;
                    i++;
                    continue;
                }
            }
            // Make sure a single user didn't do multiple alternative annotations at a single
            // position. So there is currently no support for calculating agreement on stacking
            // annotations.
            List<Configuration> cfgs = cfgSet.getConfigurations(user);
            if (cfgs.size() > 1) {
                pluralitySets.add(cfgSet);
                continue nextPosition;
            }
            Configuration cfg = cfgs.get(0);
            // Check if source and/or targets of a relation are stacked
            if (cfg.getPosition() instanceof RelationPosition) {
                RelationPosition pos = (RelationPosition) cfg.getPosition();
                FeatureStructure arc = cfg.getFs(user, pos.getCasId(), aCasMap);
                RelationDiffAdapter adapter = (RelationDiffAdapter) aDiff.getTypeAdapters().get(pos.getType());
                // Check if the source of the relation is stacked
                AnnotationFS source = FSUtil.getFeature(arc, adapter.getSourceFeature(), AnnotationFS.class);
                List<AnnotationFS> sourceCandidates = CasUtil.selectAt(arc.getCAS(), source.getType(), source.getBegin(), source.getEnd());
                if (sourceCandidates.size() > 1) {
                    pluralitySets.add(cfgSet);
                    continue nextPosition;
                }
                // Check if the target of the relation is stacked
                AnnotationFS target = FSUtil.getFeature(arc, adapter.getTargetFeature(), AnnotationFS.class);
                List<AnnotationFS> targetCandidates = CasUtil.selectAt(arc.getCAS(), target.getType(), target.getBegin(), target.getEnd());
                if (targetCandidates.size() > 1) {
                    pluralitySets.add(cfgSet);
                    continue nextPosition;
                }
            }
            // Only calculate agreement for the given feature
            FeatureStructure fs = cfg.getFs(user, cfg.getPosition().getCasId(), aCasMap);
            values[i] = extractValueForAgreement(fs, aFeature, cfg.getAID(user).index, cfg.getPosition().getLinkCompareBehavior());
            // agreement calculation. The empty label is still a valid label.
            if (aNullLabelsAsEmpty && values[i] == null) {
                values[i] = "";
            }
            // "null" cannot be used in agreement calculations. We treat these as incomplete
            if (values[i] == null) {
                incompleteSetsByLabel.add(cfgSet);
                if (aExcludeIncomplete) {
                    continue nextPosition;
                }
            }
            i++;
        }
        if (ObjectUtils.notEqual(values[0], values[1])) {
            setsWithDifferences.add(cfgSet);
        }
        // are calculating agreement over
        assert cfgSet.getPosition().getFeature() == null || cfgSet.getPosition().getFeature().equals(aFeature);
        completeSets.add(cfgSet);
        study.addItemAsArray(values);
    }
    return new CodingAgreementResult(aType, aFeature, aDiff.toResult(), study, users, completeSets, irrelevantSets, setsWithDifferences, incompleteSetsByPosition, incompleteSetsByLabel, pluralitySets, aExcludeIncomplete);
}
Also used : LinkCompareBehavior(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.LinkCompareBehavior) WebAnnoCasUtil.getFeature(de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.getFeature) CasDiff(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff) AnnotationFS(org.apache.uima.cas.text.AnnotationFS) Configuration(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.Configuration) ConfigurationSet(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.ConfigurationSet) ByteArrayOutputStream(java.io.ByteArrayOutputStream) CAS(org.apache.uima.cas.CAS) IAnnotationUnit(org.dkpro.statistics.agreement.IAnnotationUnit) FSUtil(org.apache.uima.fit.util.FSUtil) ArrayList(java.util.ArrayList) ByteArrayInputStream(java.io.ByteArrayInputStream) CSVFormat(org.apache.commons.csv.CSVFormat) Arrays.asList(java.util.Arrays.asList) ObjectUtils(org.apache.commons.lang3.ObjectUtils) Map(java.util.Map) OutputStreamWriter(java.io.OutputStreamWriter) FeatureStructure(org.apache.uima.cas.FeatureStructure) Position(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.api.Position) PrintStream(java.io.PrintStream) TypeSystem(org.apache.uima.cas.TypeSystem) CodingAnnotationStudy(org.dkpro.statistics.agreement.coding.CodingAnnotationStudy) CodingAgreementResult(de.tudarmstadt.ukp.clarin.webanno.agreement.results.coding.CodingAgreementResult) ArrayFS(org.apache.uima.cas.ArrayFS) Collection(java.util.Collection) ICodingAnnotationItem(org.dkpro.statistics.agreement.coding.ICodingAnnotationItem) IOException(java.io.IOException) RelationDiffAdapter(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationDiffAdapter) CasUtil(org.apache.uima.fit.util.CasUtil) List(java.util.List) RelationPosition(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationPosition) ICodingAnnotationStudy(org.dkpro.statistics.agreement.coding.ICodingAnnotationStudy) Collections(java.util.Collections) CSVPrinter(org.apache.commons.csv.CSVPrinter) InputStream(java.io.InputStream) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) TypeSystem(org.apache.uima.cas.TypeSystem) Configuration(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.Configuration) Position(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.api.Position) RelationPosition(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationPosition) ArrayList(java.util.ArrayList) CodingAnnotationStudy(org.dkpro.statistics.agreement.coding.CodingAnnotationStudy) ICodingAnnotationStudy(org.dkpro.statistics.agreement.coding.ICodingAnnotationStudy) CodingAgreementResult(de.tudarmstadt.ukp.clarin.webanno.agreement.results.coding.CodingAgreementResult) FeatureStructure(org.apache.uima.cas.FeatureStructure) ConfigurationSet(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.ConfigurationSet) AnnotationFS(org.apache.uima.cas.text.AnnotationFS) CAS(org.apache.uima.cas.CAS) RelationDiffAdapter(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationDiffAdapter) ArrayList(java.util.ArrayList) Arrays.asList(java.util.Arrays.asList) List(java.util.List) RelationPosition(de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationPosition)

Aggregations

CodingAgreementResult (de.tudarmstadt.ukp.clarin.webanno.agreement.results.coding.CodingAgreementResult)1 WebAnnoCasUtil.getFeature (de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.getFeature)1 CasDiff (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff)1 Configuration (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.Configuration)1 ConfigurationSet (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.CasDiff.ConfigurationSet)1 LinkCompareBehavior (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.LinkCompareBehavior)1 Position (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.api.Position)1 RelationDiffAdapter (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationDiffAdapter)1 RelationPosition (de.tudarmstadt.ukp.clarin.webanno.curation.casdiff.relation.RelationPosition)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 OutputStreamWriter (java.io.OutputStreamWriter)1 PrintStream (java.io.PrintStream)1 ArrayList (java.util.ArrayList)1 Arrays.asList (java.util.Arrays.asList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 List (java.util.List)1