Examples with EvaluationResult de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult used on opensource projects

Example 1 with EvaluationResult

use of de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult in project inception by inception-project.

the class OpenNlpNerRecommender method evaluate.

@Override
public EvaluationResult evaluate(List<CAS> aCasses, DataSplitter aDataSplitter) throws RecommendationException {
    List<NameSample> data = extractNameSamples(aCasses);
    List<NameSample> trainingSet = new ArrayList<>();
    List<NameSample> testSet = new ArrayList<>();
    for (NameSample nameSample : data) {
        switch(aDataSplitter.getTargetSet(nameSample)) {
            case TRAIN:
                trainingSet.add(nameSample);
                break;
            case TEST:
                testSet.add(nameSample);
                break;
            default:
                // Do nothing
                break;
        }
    }
    int testSetSize = testSet.size();
    int trainingSetSize = trainingSet.size();
    double overallTrainingSize = data.size() - testSetSize;
    double trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0;
    final int minTrainingSetSize = 2;
    final int minTestSetSize = 2;
    if (trainingSetSize < minTrainingSetSize || testSetSize < minTestSetSize) {
        if ((getRecommender().getThreshold() <= 0.0d)) {
            return new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName());
        }
        String info = String.format("Not enough evaluation data: training set [%s] sentences, test set [%s] of total [%s]", trainingSetSize, testSetSize, data.size());
        LOG.info(info);
        EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio);
        result.setEvaluationSkipped(true);
        result.setErrorMsg(info);
        return result;
    }
    LOG.info("Training on [{}] sentences, predicting on [{}] of total [{}]", trainingSet.size(), testSet.size(), data.size());
    // Train model
    TokenNameFinderModel model = train(trainingSet, traits.getParameters());
    NameFinderME nameFinder = new NameFinderME(model);
    // Evaluate
    List<LabelPair> labelPairs = new ArrayList<>();
    for (NameSample sample : testSet) {
        // clear adaptive data from feature generators if necessary
        if (sample.isClearAdaptiveDataSet()) {
            nameFinder.clearAdaptiveData();
        }
        // Span contains one NE, Array of them all in one sentence
        String[] sentence = sample.getSentence();
        Span[] predictedNames = nameFinder.find(sentence);
        Span[] goldNames = sample.getNames();
        labelPairs.addAll(determineLabelsForASentence(sentence, predictedNames, goldNames));
    }
    return labelPairs.stream().collect(toEvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio, NO_NE_TAG));
}

Example 2 with EvaluationResult

use of de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult in project inception by inception-project.

the class OpenNlpNerRecommenderTest method thatEvaluationWorks.

@Test
public void thatEvaluationWorks() throws Exception {
    DataSplitter splitStrategy = new PercentageBasedSplitter(0.8, 10);
    OpenNlpNerRecommender sut = new OpenNlpNerRecommender(recommender, traits);
    List<CAS> casList = loadDevelopmentData();
    EvaluationResult result = sut.evaluate(casList, splitStrategy);
    double fscore = result.computeF1Score();
    double accuracy = result.computeAccuracyScore();
    double precision = result.computePrecisionScore();
    double recall = result.computeRecallScore();
    System.out.printf("F1-Score: %f%n", fscore);
    System.out.printf("Accuracy: %f%n", accuracy);
    System.out.printf("Precision: %f%n", precision);
    System.out.printf("Recall: %f%n", recall);
    assertThat(fscore).isStrictlyBetween(0.0, 1.0);
    assertThat(precision).isStrictlyBetween(0.0, 1.0);
    assertThat(recall).isStrictlyBetween(0.0, 1.0);
    assertThat(accuracy).isStrictlyBetween(0.0, 1.0);
}

Example 3 with EvaluationResult

use of de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult in project inception by inception-project.

the class OpenNlpPosRecommenderTest method thatEvaluationWorks.

@Test
public void thatEvaluationWorks() throws Exception {
    DataSplitter splitStrategy = new PercentageBasedSplitter(0.8, 10);
    OpenNlpPosRecommender sut = new OpenNlpPosRecommender(recommender, traits);
    List<CAS> casList = loadDevelopmentData();
    EvaluationResult result = sut.evaluate(casList, splitStrategy);
    double fscore = result.computeF1Score();
    double accuracy = result.computeAccuracyScore();
    double precision = result.computePrecisionScore();
    double recall = result.computeRecallScore();
    System.out.printf("F1-Score: %f%n", fscore);
    System.out.printf("Accuracy: %f%n", accuracy);
    System.out.printf("Precision: %f%n", precision);
    System.out.printf("Recall: %f%n", recall);
    assertThat(fscore).isStrictlyBetween(0.0, 1.0);
    assertThat(precision).isStrictlyBetween(0.0, 1.0);
    assertThat(recall).isStrictlyBetween(0.0, 1.0);
    assertThat(accuracy).isStrictlyBetween(0.0, 1.0);
}

Example 4 with EvaluationResult

use of de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult in project inception by inception-project.

the class NamedEntityLinker method evaluate.

@Override
public EvaluationResult evaluate(List<CAS> aCasses, DataSplitter aDataSplitter) {
    EvaluationResult result = new EvaluationResult();
    result.setEvaluationSkipped(true);
    result.setErrorMsg("NamedEntityLinker does not support evaluation.");
    return result;
}

Example 5 with EvaluationResult

use of de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult in project inception by inception-project.

the class DataMajorityNerRecommender method evaluate.

// end::predict2[]
// tag::evaluate[]
@Override
public EvaluationResult evaluate(List<CAS> aCasses, DataSplitter aDataSplitter) throws RecommendationException {
    List<Annotation> data = extractAnnotations(aCasses);
    List<Annotation> trainingData = new ArrayList<>();
    List<Annotation> testData = new ArrayList<>();
    for (Annotation ann : data) {
        switch(aDataSplitter.getTargetSet(ann)) {
            case TRAIN:
                trainingData.add(ann);
                break;
            case TEST:
                testData.add(ann);
                break;
            case IGNORE:
                break;
        }
    }
    int trainingSetSize = trainingData.size();
    int testSetSize = testData.size();
    double overallTrainingSize = data.size() - testSetSize;
    double trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0;
    if (trainingData.size() < 1 || testData.size() < 1) {
        log.info("Not enough data to evaluate, skipping!");
        EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), getRecommender().getLayer().getUiName(), trainingSetSize, testSetSize, trainRatio);
        result.setEvaluationSkipped(true);
        return result;
    }
    DataMajorityModel model = trainModel(trainingData);
    // evaluation: collect predicted and gold labels for evaluation
    EvaluationResult result = testData.stream().map(anno -> new LabelPair(anno.label, model.majorityLabel)).collect(toEvaluationResult(DATAPOINT_UNIT.getSimpleName(), getRecommender().getLayer().getUiName(), trainingSetSize, testSetSize, trainRatio));
    return result;
}