Examples with RecommenderEvaluationResultEvent de.tudarmstadt.ukp.inception.recommendation.event.RecommenderEvaluationResultEvent used on opensource projects

Example 1 with RecommenderEvaluationResultEvent

use of de.tudarmstadt.ukp.inception.recommendation.event.RecommenderEvaluationResultEvent in project inception by inception-project.

the class SelectionTask method execute.

@Override
public void execute() {
    try (CasStorageSession session = CasStorageSession.open()) {
        Project project = getProject();
        User user = getUser().orElseThrow();
        String userName = user.getUsername();
        // Read the CASes only when they are accessed the first time. This allows us to skip
        // reading the CASes in case that no layer / recommender is available or if no
        // recommender requires evaluation.
        LazyInitializer<List<CAS>> casses = new LazyInitializer<List<CAS>>() {

            @Override
            protected List<CAS> initialize() {
                return readCasses(project, userName);
            }
        };
        boolean seenRecommender = false;
        for (AnnotationLayer layer : annoService.listAnnotationLayer(getProject())) {
            if (!layer.isEnabled()) {
                continue;
            }
            List<Recommender> recommenders = recommendationService.listRecommenders(layer);
            if (recommenders == null || recommenders.isEmpty()) {
                log.trace("[{}][{}]: No recommenders, skipping selection.", userName, layer.getUiName());
                continue;
            }
            seenRecommender = true;
            List<EvaluatedRecommender> evaluatedRecommenders = new ArrayList<>();
            for (Recommender r : recommenders) {
                // Make sure we have the latest recommender config from the DB - the one from
                // the active recommenders list may be outdated
                Recommender recommender;
                try {
                    recommender = recommendationService.getRecommender(r.getId());
                } catch (NoResultException e) {
                    log.info("[{}][{}]: Recommender no longer available... skipping", user.getUsername(), r.getName());
                    continue;
                }
                if (!recommender.isEnabled()) {
                    log.debug("[{}][{}]: Disabled - skipping", userName, recommender.getName());
                    continue;
                }
                String recommenderName = recommender.getName();
                try {
                    long start = System.currentTimeMillis();
                    RecommendationEngineFactory<?> factory = recommendationService.getRecommenderFactory(recommender).orElse(null);
                    if (factory == null) {
                        log.error("[{}][{}]: No recommender factory available for [{}]", user.getUsername(), r.getName(), r.getTool());
                        appEventPublisher.publishEvent(new SelectionTaskEvent(this, recommender, user.getUsername(), String.format("No recommender factory available for %s", recommender.getTool())));
                        continue;
                    }
                    if (!factory.accepts(recommender.getLayer(), recommender.getFeature())) {
                        log.info("[{}][{}]: Recommender configured with invalid layer or feature " + "- skipping recommender", user.getUsername(), r.getName());
                        logMessages.add(info(this, "Recommender [%s] configured with invalid layer or feature - skipping recommender", recommenderName));
                        evaluatedRecommenders.add(EvaluatedRecommender.makeInactiveWithoutEvaluation(recommender, "Invalid layer or feature"));
                        continue;
                    }
                    RecommendationEngine recommendationEngine = factory.build(recommender);
                    if (recommender.isAlwaysSelected()) {
                        log.debug("[{}][{}]: Activating [{}] without evaluating - always selected", userName, recommenderName, recommenderName);
                        logMessages.add(info(this, "Recommender [%s] activated without evaluating - always selected", recommenderName));
                        evaluatedRecommenders.add(EvaluatedRecommender.makeActiveWithoutEvaluation(recommender));
                        continue;
                    } else if (!factory.isEvaluable()) {
                        log.debug("[{}][{}]: Activating [{}] without evaluating - not evaluable", userName, recommenderName, recommenderName);
                        logMessages.add(info(this, "Recommender [%s] activated without evaluating - not evaluable", recommenderName));
                        evaluatedRecommenders.add(EvaluatedRecommender.makeActiveWithoutEvaluation(recommender));
                        continue;
                    }
                    log.info("[{}][{}]: Evaluating...", userName, recommenderName);
                    DataSplitter splitter = new PercentageBasedSplitter(0.8, 10);
                    EvaluationResult result = recommendationEngine.evaluate(casses.get(), splitter);
                    if (result.isEvaluationSkipped()) {
                        String msg = String.format("Evaluation of recommender [%s] could not be performed: %s", recommenderName, result.getErrorMsg().orElse("unknown reason"));
                        log.info("[{}][{}]: {}", user.getUsername(), recommenderName, msg);
                        logMessages.add(LogMessage.warn(this, "%s", msg));
                        evaluatedRecommenders.add(EvaluatedRecommender.makeInactiveWithoutEvaluation(recommender, msg));
                        continue;
                    }
                    double score = result.computeF1Score();
                    double threshold = recommender.getThreshold();
                    boolean activated;
                    if (score >= threshold) {
                        activated = true;
                        evaluatedRecommenders.add(EvaluatedRecommender.makeActive(recommender, result, format("Score {0,number,#.####} >= threshold {1,number,#.####}", score, threshold)));
                        log.info("[{}][{}]: Activated ({} >= threshold {})", user.getUsername(), recommenderName, score, threshold);
                        logMessages.add(info(this, "Recommender [%s] activated (%f >= threshold %f)", recommenderName, score, threshold));
                    } else {
                        activated = false;
                        log.info("[{}][{}]: Not activated ({} < threshold {})", user.getUsername(), recommenderName, score, threshold);
                        logMessages.add(info(this, "Recommender [%s] not activated (%f < threshold %f)", recommenderName, score, threshold));
                        evaluatedRecommenders.add(EvaluatedRecommender.makeInactive(recommender, result, format("Score {0,number,#.####} < threshold {1,number,#.####}", score, threshold)));
                    }
                    appEventPublisher.publishEvent(new RecommenderEvaluationResultEvent(this, recommender, user.getUsername(), result, System.currentTimeMillis() - start, activated));
                    Optional<String> recError = result.getErrorMsg();
                    SelectionTaskEvent evalEvent = new SelectionTaskEvent(this, recommender, user.getUsername(), result);
                    if (recError.isPresent()) {
                        evalEvent.setErrorMsg(recError.get());
                    }
                    appEventPublisher.publishEvent(evalEvent);
                }// even if a particular recommender fails.
                 catch (Throwable e) {
                    log.error("[{}][{}]: Failed", user.getUsername(), recommenderName, e);
                    appEventPublisher.publishEvent(new SelectionTaskEvent(this, recommender, user.getUsername(), e.getMessage()));
                }
            }
            recommendationService.setEvaluatedRecommenders(user, layer, evaluatedRecommenders);
        }
        if (!seenRecommender) {
            log.trace("[{}]: No recommenders configured, skipping training.", userName);
            return;
        }
        if (!recommendationService.hasActiveRecommenders(user.getUsername(), project)) {
            log.debug("[{}]: No recommenders active, skipping training.", userName);
            return;
        }
        TrainingTask trainingTask = new TrainingTask(user, getProject(), "SelectionTask after activating recommenders", currentDocument);
        trainingTask.inheritLog(logMessages);
        schedulingService.enqueue(trainingTask);
    }
}