Examples with TrainingCapability de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability used on opensource projects

Example 1 with TrainingCapability

use of de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability in project inception by inception-project.

the class TrainingTask method execute.

@Override
public void execute() {
    try (CasStorageSession session = CasStorageSession.open()) {
        Project project = getProject();
        User user = getUser().orElseThrow();
        log.debug("[{}][{}]: Starting training for project {} triggered by [{}]...", getId(), user.getUsername(), project, getTrigger());
        logMessages.add(info(this, "Starting training triggered by [%s]...", getTrigger()));
        // Read the CASes only when they are accessed the first time. This allows us to skip
        // reading the CASes in case that no layer / recommender is available or if no
        // recommender requires evaluation.
        LazyInitializer<List<TrainingDocument>> casses = new LazyInitializer<List<TrainingDocument>>() {

            @Override
            protected List<TrainingDocument> initialize() {
                return readCasses(project, user);
            }
        };
        boolean seenSuccessfulTraining = false;
        boolean seenNonTrainingRecommender = false;
        for (AnnotationLayer layer : annoService.listAnnotationLayer(project)) {
            if (!layer.isEnabled()) {
                continue;
            }
            List<EvaluatedRecommender> recommenders = recommendationService.getActiveRecommenders(user, layer);
            if (recommenders.isEmpty()) {
                log.trace("[{}][{}][{}]: No active recommenders, skipping training.", getId(), user.getUsername(), layer.getUiName());
                logMessages.add(info(this, "No active recommenders for layer [%s], skipping training.", layer.getUiName()));
                continue;
            }
            for (EvaluatedRecommender r : recommenders) {
                // Make sure we have the latest recommender config from the DB - the one from
                // the active recommenders list may be outdated
                Recommender recommender;
                try {
                    recommender = recommendationService.getRecommender(r.getRecommender().getId());
                } catch (NoResultException e) {
                    log.debug("[{}][{}][{}]: Recommender no longer available... skipping", getId(), user.getUsername(), r.getRecommender().getName());
                    continue;
                }
                if (!recommender.isEnabled()) {
                    log.debug("[{}][{}][{}]: Disabled - skipping", user.getUsername(), getId(), r.getRecommender().getName());
                    continue;
                }
                long startTime = System.currentTimeMillis();
                try {
                    Optional<RecommendationEngineFactory<?>> maybeFactory = recommendationService.getRecommenderFactory(recommender);
                    if (maybeFactory.isEmpty()) {
                        log.warn("[{}][{}]: No factory found - skipping recommender", user.getUsername(), r.getRecommender().getName());
                        continue;
                    }
                    RecommendationEngineFactory<?> factory = maybeFactory.get();
                    if (!factory.accepts(recommender.getLayer(), recommender.getFeature())) {
                        log.debug("[{}][{}][{}]: Recommender configured with invalid layer or " + "feature - skipping recommender", getId(), user.getUsername(), r.getRecommender().getName());
                        logMessages.add(error(this, "Recommender [%s] configured with invalid layer or feature - skipping recommender.", r.getRecommender().getName()));
                        appEventPublisher.publishEvent(new RecommenderTaskEvent(this, user.getUsername(), "Recommender configured with invalid layer or feature - skipping training recommender.", recommender));
                        continue;
                    }
                    RecommendationEngine recommendationEngine = factory.build(recommender);
                    RecommenderContext ctx = recommendationEngine.newContext(recommendationService.getContext(user, recommender).orElse(RecommenderContext.EMPTY_CONTEXT));
                    ctx.setUser(user);
                    TrainingCapability capability = recommendationEngine.getTrainingCapability();
                    // prediction
                    if (capability == TRAINING_NOT_SUPPORTED) {
                        seenNonTrainingRecommender = true;
                        log.debug("[{}][{}][{}]: Engine does not support training", getId(), user.getUsername(), recommender.getName());
                        ctx.close();
                        recommendationService.putContext(user, recommender, ctx);
                        continue;
                    }
                    List<CAS> cassesForTraining = // 
                    casses.get().stream().filter(e -> !recommender.getStatesIgnoredForTraining().contains(e.state)).filter(e -> containsTargetTypeAndFeature(recommender, e.cas)).map(e -> e.cas).collect(toList());
                    // do not mark as ready
                    if (cassesForTraining.isEmpty() && capability == TRAINING_REQUIRED) {
                        log.debug("[{}][{}][{}]: There are no annotations available to train on", getId(), user.getUsername(), recommender.getName());
                        logMessages.add(warn(this, "There are no [%s] annotations available to train on.", layer.getUiName()));
                        // This can happen if there were already predictions based on existing
                        // annotations, but all annotations have been removed/deleted. To ensure
                        // that the prediction run removes the stale predictions, we need to
                        // call it a success here.
                        seenSuccessfulTraining = true;
                        continue;
                    }
                    log.debug("[{}][{}][{}]: Training model on [{}] out of [{}] documents ...", getId(), user.getUsername(), recommender.getName(), cassesForTraining.size(), casses.get().size());
                    logMessages.add(info(this, "Training model for [%s] on [%d] out of [%d] documents ...", layer.getUiName(), cassesForTraining.size(), casses.get().size()));
                    recommendationEngine.train(ctx, cassesForTraining);
                    logMessages.addAll(ctx.getMessages());
                    long duration = System.currentTimeMillis() - startTime;
                    if (!recommendationEngine.isReadyForPrediction(ctx)) {
                        int docNum = casses.get().size();
                        int trainDocNum = cassesForTraining.size();
                        log.debug("[{}][{}][{}]: Training on [{}] out of [{}] documents not successful ({} ms)", getId(), user.getUsername(), recommender.getName(), trainDocNum, docNum, duration);
                        logMessages.add(info(this, "Training not successful (%d ms).", duration));
                        appEventPublisher.publishEvent(new RecommenderTaskEvent(this, user.getUsername(), format("Training on %d out of %d documents not successful (%d ms)", trainDocNum, docNum, duration), recommender));
                        continue;
                    }
                    log.debug("[{}][{}][{}]: Training successful on [{}] out of [{}] documents ({} ms)", getId(), user.getUsername(), recommender.getName(), cassesForTraining.size(), casses.get().size(), duration);
                    logMessages.add(info(this, "Training successful on [%d] out of [%d] documents (%d ms)", cassesForTraining.size(), casses.get().size(), duration));
                    seenSuccessfulTraining = true;
                    ctx.close();
                    recommendationService.putContext(user, recommender, ctx);
                }// even if a particular recommender fails.
                 catch (Throwable e) {
                    long duration = System.currentTimeMillis() - startTime;
                    log.error("[{}][{}][{}]: Training failed ({} ms)", getId(), user.getUsername(), recommender.getName(), (System.currentTimeMillis() - startTime), e);
                    logMessages.add(error(this, "Training failed (%d ms): %s", duration, getRootCauseMessage(e)));
                    appEventPublisher.publishEvent(new RecommenderTaskEvent(this, user.getUsername(), String.format("Training failed (%d ms) with %s", duration, e.getMessage()), recommender));
                }
            }
        }
        if (!seenSuccessfulTraining && !seenNonTrainingRecommender) {
            log.debug("[{}][{}]: No recommenders trained successfully and no non-training " + "recommenders, skipping prediction.", getId(), user.getUsername());
            return;
        }
        PredictionTask predictionTask = new PredictionTask(user, String.format("TrainingTask %s complete", getId()), currentDocument);
        predictionTask.inheritLog(logMessages);
        schedulingService.enqueue(predictionTask);
    }
}