Examples with Project de.tudarmstadt.ukp.clarin.webanno.model.Project used on opensource projects

Example 31 with Project

use of de.tudarmstadt.ukp.clarin.webanno.model.Project in project webanno by webanno.

the class AnnotationSchemaServiceImpl method createTag.

@Override
@Transactional
public void createTag(Tag aTag) {
    if (isNull(aTag.getId())) {
        entityManager.persist(aTag);
    } else {
        entityManager.merge(aTag);
    }
    try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(aTag.getTagSet().getProject().getId()))) {
        TagSet tagset = aTag.getTagSet();
        Project project = tagset.getProject();
        log.info("Created tag [{}]({}) in tagset [{}]({}) in project [{}]({})", aTag.getName(), aTag.getId(), tagset.getName(), tagset.getId(), project.getName(), project.getId());
    }
}

Example 32 with Project

use of de.tudarmstadt.ukp.clarin.webanno.model.Project in project webanno by webanno.

the class CasStorageServiceImpl method realWriteCas.

private void realWriteCas(Project aProject, String aDocumentName, long aDocumentId, JCas aJcas, String aUserName, File aAnnotationFolder, File aTargetPath) throws IOException {
    log.debug("Writing annotation document [{}]({}) for user [{}] in project [{}]({})", aDocumentName, aDocumentId, aUserName, aProject.getName(), aProject.getId());
    try {
        if (casDoctor != null) {
            casDoctor.analyze(aProject, aJcas.getCas());
        }
    } catch (CasDoctorException e) {
        StringBuilder detailMsg = new StringBuilder();
        detailMsg.append("CAS Doctor found problems for user [").append(aUserName).append("] in source document [").append(aDocumentName).append("] (").append(aDocumentId).append(") in project[").append(aProject.getName()).append("] (").append(aProject.getId()).append(")\n");
        e.getDetails().forEach(m -> detailMsg.append(String.format("- [%s] %s%n", m.level, m.message)));
        throw new DataRetrievalFailureException(detailMsg.toString());
    } catch (Exception e) {
        throw new DataRetrievalFailureException("Error analyzing CAS of user [" + aUserName + "] in source document [" + aDocumentName + "] (" + aDocumentId + ") in project [" + aProject.getName() + "] (" + aProject.getId() + ")", e);
    }
    synchronized (lock) {
        // File annotationFolder = getAnnotationFolder(aDocument);
        FileUtils.forceMkdir(aAnnotationFolder);
        final String username = aUserName;
        File currentVersion = new File(aAnnotationFolder, username + ".ser");
        File oldVersion = new File(aAnnotationFolder, username + ".ser.old");
        // Save current version
        try {
            // Make a backup of the current version of the file before overwriting
            if (currentVersion.exists()) {
                renameFile(currentVersion, oldVersion);
            }
            // Now write the new version to "<username>.ser" or CURATION_USER.ser
            DocumentMetaData md;
            try {
                md = DocumentMetaData.get(aJcas);
            } catch (IllegalArgumentException e) {
                md = DocumentMetaData.create(aJcas);
            }
            md.setDocumentId(aUserName);
            // File targetPath = getAnnotationFolder(aDocument);
            CasPersistenceUtils.writeSerializedCas(aJcas, new File(aTargetPath, aUserName + ".ser"));
            try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(aProject.getId()))) {
                log.info("Updated annotations for user [{}] on document [{}]({}) in project [{}]({})", aUserName, aDocumentName, aDocumentId, aProject.getName(), aProject.getId());
            }
            // If the saving was successful, we delete the old version
            if (oldVersion.exists()) {
                FileUtils.forceDelete(oldVersion);
            }
        } catch (IOException e) {
            // If we could not save the new version, restore the old one.
            FileUtils.forceDelete(currentVersion);
            // If this is the first version, there is no old version, so do not restore anything
            if (oldVersion.exists()) {
                renameFile(oldVersion, currentVersion);
            }
            // Now abort anyway
            throw e;
        }
        // Manage history
        if (backupInterval > 0) {
            // Determine the reference point in time based on the current version
            long now = currentVersion.lastModified();
            // Get all history files for the current user
            File[] history = aAnnotationFolder.listFiles(new FileFilter() {

                private final Matcher matcher = Pattern.compile(Pattern.quote(username) + "\\.ser\\.[0-9]+\\.bak").matcher("");

                @Override
                public boolean accept(File aFile) {
                    // Check if the filename matches the pattern given above.
                    return matcher.reset(aFile.getName()).matches();
                }
            });
            // Sort the files (oldest one first)
            Arrays.sort(history, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
            // Check if we need to make a new history file
            boolean historyFileCreated = false;
            File historyFile = new File(aAnnotationFolder, username + ".ser." + now + ".bak");
            if (history.length == 0) {
                // If there is no history yet but we should keep history, then we create a
                // history file in any case.
                FileUtils.copyFile(currentVersion, historyFile);
                historyFileCreated = true;
            } else {
                // Check if the newest history file is significantly older than the current one
                File latestHistory = history[history.length - 1];
                if (latestHistory.lastModified() + backupInterval < now) {
                    FileUtils.copyFile(currentVersion, historyFile);
                    historyFileCreated = true;
                }
            }
            // Prune history based on number of backup
            if (historyFileCreated) {
                // The new version is not in the history, so we keep that in any case. That
                // means we need to keep one less.
                int toKeep = Math.max(backupKeepNumber - 1, 0);
                if ((backupKeepNumber > 0) && (toKeep < history.length)) {
                    // Copy the oldest files to a new array
                    File[] toRemove = new File[history.length - toKeep];
                    System.arraycopy(history, 0, toRemove, 0, toRemove.length);
                    // Restrict the history to what is left
                    File[] newHistory = new File[toKeep];
                    if (toKeep > 0) {
                        System.arraycopy(history, toRemove.length, newHistory, 0, newHistory.length);
                    }
                    history = newHistory;
                    // Remove these old files
                    for (File file : toRemove) {
                        FileUtils.forceDelete(file);
                        try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(aProject.getId()))) {
                            log.info("Removed surplus history file [{}] of user [{}] for " + "document [{}]({}) in project [{}]({})", file.getName(), aUserName, aDocumentName, aDocumentId, aProject.getName(), aProject.getId());
                        }
                    }
                }
                // Prune history based on time
                if (backupKeepTime > 0) {
                    for (File file : history) {
                        if ((file.lastModified() + backupKeepTime) < now) {
                            FileUtils.forceDelete(file);
                            try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(aProject.getId()))) {
                                log.info("Removed outdated history file [{}] of user [{}] for " + "document [{}]({}) in project [{}]({})", file.getName(), aUserName, aDocumentName, aDocumentId, aProject.getName(), aProject.getId());
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 33 with Project

use of de.tudarmstadt.ukp.clarin.webanno.model.Project in project webanno by webanno.

the class DocumentServiceImpl method recalculateProjectState.

private void recalculateProjectState(Project aProject) {
    Project project;
    try {
        project = projectService.getProject(aProject.getId());
    } catch (NoResultException e) {
        // updating its state. So then we do nothing here.
        return;
    }
    String query = "SELECT new " + SourceDocumentStateStats.class.getName() + "(" + "COUNT(*), " + "SUM(CASE WHEN state = :an  THEN 1 ELSE 0 END), " + "SUM(CASE WHEN (state = :aip OR state is NULL) THEN 1 ELSE 0 END), " + "SUM(CASE WHEN state = :af  THEN 1 ELSE 0 END), " + "SUM(CASE WHEN state = :cip THEN 1 ELSE 0 END), " + "SUM(CASE WHEN state = :cf  THEN 1 ELSE 0 END)) " + "FROM SourceDocument " + "WHERE project = :project";
    SourceDocumentStateStats stats = entityManager.createQuery(query, SourceDocumentStateStats.class).setParameter("project", aProject).setParameter("an", SourceDocumentState.NEW).setParameter("aip", SourceDocumentState.ANNOTATION_IN_PROGRESS).setParameter("af", SourceDocumentState.ANNOTATION_FINISHED).setParameter("cip", SourceDocumentState.CURATION_IN_PROGRESS).setParameter("cf", SourceDocumentState.CURATION_FINISHED).getSingleResult();
    ProjectState oldState = project.getState();
    if (stats.total == stats.cf) {
        project.setState(ProjectState.CURATION_FINISHED);
    } else if (stats.total == stats.af) {
        project.setState(ProjectState.ANNOTATION_FINISHED);
    } else if (stats.total == stats.an) {
        project.setState(ProjectState.NEW);
    } else if (stats.cip > 0) {
        project.setState(ProjectState.CURATION_IN_PROGRESS);
    } else if (stats.aip > 0) {
        project.setState(ProjectState.ANNOTATION_IN_PROGRESS);
    } else {
        throw new IllegalStateException("Unable to determine project state.");
    }
    if (!Objects.equals(oldState, project.getState())) {
        applicationEventPublisher.publishEvent(new ProjectStateChangedEvent(this, project, oldState));
    }
    projectService.updateProject(project);
}

Example 34 with Project

use of de.tudarmstadt.ukp.clarin.webanno.model.Project in project webanno by webanno.

the class DocumentServiceImpl method uploadSourceDocument.

@Override
@Transactional
public void uploadSourceDocument(InputStream aIs, SourceDocument aDocument) throws IOException {
    // Create the metadata record - this also assigns the ID to the document
    createSourceDocument(aDocument);
    // Import the actual content
    File targetFile = getSourceDocumentFile(aDocument);
    JCas jcas;
    try {
        FileUtils.forceMkdir(targetFile.getParentFile());
        try (OutputStream os = new FileOutputStream(targetFile)) {
            copyLarge(aIs, os);
        }
        // Check if the file has a valid format / can be converted without error
        // This requires that the document ID has already been assigned
        jcas = createInitialCas(aDocument);
    } catch (IOException e) {
        FileUtils.forceDelete(targetFile);
        removeSourceDocument(aDocument);
        throw e;
    } catch (Exception e) {
        FileUtils.forceDelete(targetFile);
        removeSourceDocument(aDocument);
        throw new IOException(e.getMessage(), e);
    }
    applicationEventPublisher.publishEvent(new AfterDocumentCreatedEvent(this, aDocument, jcas));
    try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(aDocument.getProject().getId()))) {
        Project project = aDocument.getProject();
        log.info("Imported source document [{}]({}) to project [{}]({})", aDocument.getName(), aDocument.getId(), project.getName(), project.getId());
    }
}

Example 35 with Project

use of de.tudarmstadt.ukp.clarin.webanno.model.Project in project webanno by webanno.

the class ImportExportServiceImpl method exportCasToFile.

/**
 * A new directory is created using UUID so that every exported file will reside in its own
 * directory. This is useful as the written file can have multiple extensions based on the
 * Writer class used.
 */
@Override
public File exportCasToFile(CAS cas, SourceDocument aDocument, String aFileName, @SuppressWarnings("rawtypes") Class aWriter, boolean aStripExtension) throws IOException, UIMAException {
    // Update the source file name in case it is changed for some reason. This is necessary
    // for the writers to create the files under the correct names.
    Project project = aDocument.getProject();
    File currentDocumentUri = new File(dir.getAbsolutePath() + "/" + PROJECT_FOLDER + "/" + project.getId() + "/" + DOCUMENT_FOLDER + "/" + aDocument.getId() + "/" + SOURCE_FOLDER);
    DocumentMetaData documentMetadata = DocumentMetaData.get(cas.getJCas());
    documentMetadata.setDocumentUri(new File(currentDocumentUri, aFileName).toURI().toURL().toExternalForm());
    documentMetadata.setDocumentBaseUri(currentDocumentUri.toURI().toURL().toExternalForm());
    documentMetadata.setCollectionId(currentDocumentUri.toURI().toURL().toExternalForm());
    documentMetadata.setDocumentUri(new File(dir.getAbsolutePath() + "/" + PROJECT_FOLDER + "/" + project.getId() + "/" + DOCUMENT_FOLDER + "/" + aDocument.getId() + "/" + SOURCE_FOLDER + "/" + aFileName).toURI().toURL().toExternalForm());
    // update with the correct tagset name
    List<AnnotationFeature> features = annotationService.listAnnotationFeature(project);
    for (AnnotationFeature feature : features) {
        TagSet tagSet = feature.getTagset();
        if (tagSet == null) {
            continue;
        } else if (!feature.getLayer().getType().equals(WebAnnoConst.CHAIN_TYPE)) {
            updateCasWithTagSet(cas, feature.getLayer().getName(), tagSet.getName());
        }
    }
    File exportTempDir = File.createTempFile("webanno", "export");
    try {
        exportTempDir.delete();
        exportTempDir.mkdirs();
        AnalysisEngineDescription writer;
        if (aWriter.getName().equals("de.tudarmstadt.ukp.clarin.webanno.tsv.WebannoTsv3Writer")) {
            List<AnnotationLayer> layers = annotationService.listAnnotationLayer(aDocument.getProject());
            List<String> slotFeatures = new ArrayList<>();
            List<String> slotTargets = new ArrayList<>();
            List<String> linkTypes = new ArrayList<>();
            Set<String> spanLayers = new HashSet<>();
            Set<String> slotLayers = new HashSet<>();
            for (AnnotationLayer layer : layers) {
                if (layer.getType().contentEquals(WebAnnoConst.SPAN_TYPE)) {
                    // TSV will not use this
                    if (!annotationExists(cas, layer.getName())) {
                        continue;
                    }
                    boolean isslotLayer = false;
                    for (AnnotationFeature f : annotationService.listAnnotationFeature(layer)) {
                        if (MultiValueMode.ARRAY.equals(f.getMultiValueMode()) && LinkMode.WITH_ROLE.equals(f.getLinkMode())) {
                            isslotLayer = true;
                            slotFeatures.add(layer.getName() + ":" + f.getName());
                            slotTargets.add(f.getType());
                            linkTypes.add(f.getLinkTypeName());
                        }
                    }
                    if (isslotLayer) {
                        slotLayers.add(layer.getName());
                    } else {
                        spanLayers.add(layer.getName());
                    }
                }
            }
            spanLayers.addAll(slotLayers);
            List<String> chainLayers = new ArrayList<>();
            for (AnnotationLayer layer : layers) {
                if (layer.getType().contentEquals(WebAnnoConst.CHAIN_TYPE)) {
                    if (!chainAnnotationExists(cas, layer.getName() + "Chain")) {
                        continue;
                    }
                    chainLayers.add(layer.getName());
                }
            }
            List<String> relationLayers = new ArrayList<>();
            for (AnnotationLayer layer : layers) {
                if (layer.getType().contentEquals(WebAnnoConst.RELATION_TYPE)) {
                    // TSV will not use this
                    if (!annotationExists(cas, layer.getName())) {
                        continue;
                    }
                    relationLayers.add(layer.getName());
                }
            }
            writer = createEngineDescription(aWriter, JCasFileWriter_ImplBase.PARAM_TARGET_LOCATION, exportTempDir, JCasFileWriter_ImplBase.PARAM_STRIP_EXTENSION, aStripExtension, "spanLayers", spanLayers, "slotFeatures", slotFeatures, "slotTargets", slotTargets, "linkTypes", linkTypes, "chainLayers", chainLayers, "relationLayers", relationLayers);
        } else {
            writer = createEngineDescription(aWriter, JCasFileWriter_ImplBase.PARAM_TARGET_LOCATION, exportTempDir, JCasFileWriter_ImplBase.PARAM_STRIP_EXTENSION, aStripExtension);
        }
        runPipeline(cas, writer);
        // If the writer produced more than one file, we package it up as a ZIP file
        File exportFile;
        if (exportTempDir.listFiles().length > 1) {
            exportFile = new File(exportTempDir.getAbsolutePath() + ".zip");
            try {
                ZipUtils.zipFolder(exportTempDir, exportFile);
            } catch (Exception e) {
                try (MDC.MDCCloseable closable = MDC.putCloseable(Logging.KEY_PROJECT_ID, String.valueOf(project.getId()))) {
                    log.info("Unable to create zip File");
                }
            }
        } else {
            exportFile = new File(exportTempDir.getParent(), exportTempDir.listFiles()[0].getName());
            FileUtils.copyFile(exportTempDir.listFiles()[0], exportFile);
        }
        return exportFile;
    } finally {
        if (exportTempDir != null) {
            FileUtils.forceDelete(exportTempDir);
        }
    }
}