Examples with DefaultAgreementTraits de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits used on opensource projects

Example 1 with DefaultAgreementTraits

use of de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits in project webanno by webanno.

the class PairwiseCodingAgreementTable method exportAllAgreements.

private IResourceStream exportAllAgreements() {
    return new AbstractResourceStream() {

        private static final long serialVersionUID = 1L;

        @Override
        public InputStream getInputStream() throws ResourceStreamNotFoundException {
            AnnotationFeature feature = getModelObject().getFeature();
            DefaultAgreementTraits traits = getModelObject().getTraits();
            Map<String, List<CAS>> casMap = casMapSupplier.get();
            List<DiffAdapter> adapters = CasDiff.getDiffAdapters(annotationService, asList(feature.getLayer()));
            CasDiff diff = doDiff(adapters, traits.getLinkCompareBehavior(), casMap);
            // AgreementResult agreementResult = AgreementUtils.makeStudy(diff,
            // feature.getLayer().getName(), feature.getName(),
            // pref.excludeIncomplete, casMap);
            // TODO: for the moment, we always include incomplete annotations during this
            // export.
            CodingAgreementResult agreementResult = makeCodingStudy(diff, feature.getLayer().getName(), feature.getName(), false, casMap);
            try {
                return AgreementUtils.generateCsvReport(agreementResult);
            } catch (Exception e) {
                // FIXME Is there some better error handling here?
                LOG.error("Unable to generate report", e);
                throw new ResourceStreamNotFoundException(e);
            }
        }

        @Override
        public void close() throws IOException {
        // Nothing to do
        }
    };
}

Example 2 with DefaultAgreementTraits

use of de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits in project webanno by webanno.

the class AgreementPage method getCasMap.

public Map<String, List<CAS>> getCasMap() {
    if (agreementForm.featureList.getModelObject() == null) {
        return Collections.emptyMap();
    }
    Project project = agreementForm.featureList.getModelObject().getProject();
    DefaultAgreementTraits traits = (DefaultAgreementTraits) agreementForm.traitsContainer.get(MID_TRAITS).getDefaultModelObject();
    // Avoid reloading the CASes when switching features within the same project
    if (cachedCASes != null && project.equals(cachedProject) && cachedLimitToFinishedDocuments == traits.isLimitToFinishedDocuments()) {
        return cachedCASes;
    }
    List<User> users = projectService.listProjectUsersWithPermissions(project, ANNOTATOR);
    List<SourceDocument> sourceDocuments = documentService.listSourceDocuments(project);
    cachedCASes = new LinkedHashMap<>();
    for (User user : users) {
        List<CAS> cases = new ArrayList<>();
        // Bulk-fetch all source documents for which there is already an annotation document for
        // the user which is faster then checking for their existence individually
        List<SourceDocument> docsForUser = documentService.listAnnotationDocuments(project, user).stream().map(AnnotationDocument::getDocument).distinct().collect(Collectors.toList());
        nextDocument: for (SourceDocument document : sourceDocuments) {
            CAS cas = null;
            try {
                if (docsForUser.contains(document)) {
                    AnnotationDocument annotationDocument = documentService.getAnnotationDocument(document, user);
                    if (traits.isLimitToFinishedDocuments() && !annotationDocument.getState().equals(FINISHED)) {
                        // Add a skip marker (null) for the current CAS to the CAS list - this
                        // is necessary because we expect the CAS lists for all users to have
                        // the same size
                        cases.add(null);
                        continue nextDocument;
                    }
                }
                // Reads the user's annotation document or the initial source document -
                // depending on what is available
                cas = documentService.readAnnotationCas(document, user.getUsername(), AUTO_CAS_UPGRADE, SHARED_READ_ONLY_ACCESS);
            } catch (Exception e) {
                error("Unable to load data: " + ExceptionUtils.getRootCauseMessage(e));
                LOG.error("Unable to load data", e);
            }
            if (cas != null) {
                // Set the CAS name in the DocumentMetaData so that we can pick it
                // up in the Diff position for the purpose of debugging / transparency.
                FeatureStructure dmd = WebAnnoCasUtil.getDocumentMetadata(cas);
                FSUtil.setFeature(dmd, "documentId", document.getName());
                FSUtil.setFeature(dmd, "collectionId", document.getProject().getName());
            }
            // The next line can enter null values into the list if a user didn't work on this
            // source document yet.
            cases.add(cas);
        }
        cachedCASes.put(user.getUsername(), cases);
    }
    cachedProject = project;
    cachedLimitToFinishedDocuments = traits.isLimitToFinishedDocuments();
    return cachedCASes;
}

Example 3 with DefaultAgreementTraits

use of de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits in project webanno by webanno.

the class CohenKappaAgreementMeasure method calculatePairAgreement.

@Override
public CodingAgreementResult calculatePairAgreement(Map<String, List<CAS>> aCasMap) {
    AnnotationFeature feature = getFeature();
    DefaultAgreementTraits traits = getTraits();
    List<DiffAdapter> adapters = getDiffAdapters(annotationService, asList(feature.getLayer()));
    CasDiff diff = doDiff(adapters, traits.getLinkCompareBehavior(), aCasMap);
    CodingAgreementResult agreementResult = makeCodingStudy(diff, feature.getLayer().getName(), feature.getName(), true, aCasMap);
    IAgreementMeasure agreement = new CohenKappaAgreement(agreementResult.getStudy());
    if (agreementResult.getStudy().getItemCount() > 0) {
        agreementResult.setAgreement(agreement.calculateAgreement());
    } else {
        agreementResult.setAgreement(Double.NaN);
    }
    return agreementResult;
}

Example 4 with DefaultAgreementTraits

use of de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits in project webanno by webanno.

the class FleissKappaAgreementMeasure method calculatePairAgreement.

@Override
public CodingAgreementResult calculatePairAgreement(Map<String, List<CAS>> aCasMap) {
    AnnotationFeature feature = getFeature();
    DefaultAgreementTraits traits = getTraits();
    List<DiffAdapter> adapters = getDiffAdapters(annotationService, asList(feature.getLayer()));
    CasDiff diff = doDiff(adapters, traits.getLinkCompareBehavior(), aCasMap);
    CodingAgreementResult agreementResult = makeCodingStudy(diff, feature.getLayer().getName(), feature.getName(), true, aCasMap);
    IAgreementMeasure agreement = new FleissKappaAgreement(agreementResult.getStudy());
    if (agreementResult.getStudy().getItemCount() > 0) {
        agreementResult.setAgreement(agreement.calculateAgreement());
    } else {
        agreementResult.setAgreement(Double.NaN);
    }
    return agreementResult;
}