Examples with EvaluationData de.unidue.ltl.evaluation.core.EvaluationData used on opensource projects

Example 1 with EvaluationData

use of de.unidue.ltl.evaluation.core.EvaluationData in project dkpro-tc by dkpro.

the class Tc2LtlabEvalConverter method convertMultiLabelModeId2OutcomeUseInteger.

/**
 * Loads a multi-label DKPro TC id2outcome file into the evaluation data format. The values are
 * not mapped to their label names, the integer representation is used instead. This is
 * necessary for some evaluation metrics which work on the integer values
 *
 * @param id2OutcomeFile
 *            the id2outcome file
 * @return an evaluation data object
 * @throws Exception
 *             in case of error
 */
public static EvaluationData<Integer> convertMultiLabelModeId2OutcomeUseInteger(File id2OutcomeFile) throws Exception {
    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(id2OutcomeFile), "utf-8"));
    // pop first line
    reader.readLine();
    // pop header
    reader.readLine();
    EvaluationData<Integer> data = new EvaluationData<>();
    String line = null;
    while ((line = reader.readLine()) != null) {
        if (skipLine(line)) {
            continue;
        }
        int lastIdx = line.lastIndexOf("=");
        checkIndexRange(line, lastIdx);
        String docName = line.substring(0, lastIdx);
        String values = line.substring(lastIdx + 1);
        String[] valSplit = values.split(";");
        Double threshold = Double.valueOf(valSplit[2]);
        String prediction = valSplit[0];
        List<Integer> mappedPred = convertMultiLabelToIntegerArray(prediction.split(","), threshold);
        String gold = valSplit[1];
        List<Integer> mappedGold = convertMultiLabelToIntegerArray(gold.split(","), threshold);
        data.registerMultiLabel(mappedGold, mappedPred, docName);
    }
    reader.close();
    return data;
}

Example 2 with EvaluationData

use of de.unidue.ltl.evaluation.core.EvaluationData in project dkpro-tc by dkpro.

the class Tc2LtlabEvalConverter method convertRegressionModeId2Outcome.

/**
 * Loads a regression DKPro TC id2outcome file into the evaluation data format
 *
 * @param id2OutcomeFile
 *            the id2outcome file
 * @return an evaluation data object
 * @throws Exception
 *             in case of error
 */
public static EvaluationData<Double> convertRegressionModeId2Outcome(File id2OutcomeFile) throws Exception {
    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(id2OutcomeFile), "utf-8"));
    // pop head line
    reader.readLine();
    // pop header (not needed for regression)
    reader.readLine();
    EvaluationData<Double> data = new EvaluationData<>();
    String line = null;
    while ((line = reader.readLine()) != null) {
        if (line.isEmpty() || line.startsWith("#")) {
            continue;
        }
        String[] split = line.split("=");
        String docName = split[0];
        String values = split[1];
        String[] valSplit = values.split(";");
        Double prediction = Double.valueOf(valSplit[0]);
        Double gold = Double.valueOf(valSplit[1]);
        data.register(gold, prediction, docName);
    }
    reader.close();
    return data;
}

Example 3 with EvaluationData

use of de.unidue.ltl.evaluation.core.EvaluationData in project dkpro-tc by dkpro.

the class ScatterplotReport method execute.

@Override
public void execute() throws Exception {
    for (TaskContextMetadata subcontext : getSubtasks()) {
        if (TcTaskTypeUtil.isCrossValidationTask(getContext().getStorageService(), subcontext.getId())) {
            File id2outcomeFile = getContext().getStorageService().locateKey(subcontext.getId(), Constants.FILE_COMBINED_ID_OUTCOME_KEY);
            EvaluationData<Double> data = Tc2LtlabEvalConverter.convertRegressionModeId2Outcome(id2outcomeFile);
            double[] gold = new double[(int) data.size()];
            double[] prediction = new double[(int) data.size()];
            Iterator<EvaluationEntry<Double>> iterator = data.iterator();
            int i = 0;
            while (iterator.hasNext()) {
                EvaluationEntry<Double> next = iterator.next();
                gold[i] = next.getGold();
                prediction[i] = next.getPredicted();
                i++;
            }
            ScatterplotRenderer renderer = new ScatterplotRenderer(gold, prediction);
            getContext().storeBinary("scatterplot.pdf", renderer);
        } else if (TcTaskTypeUtil.isMachineLearningAdapterTask(getContext().getStorageService(), subcontext.getId())) {
            File id2outcomeFile = getContext().getStorageService().locateKey(subcontext.getId(), Constants.ID_OUTCOME_KEY);
            EvaluationData<Double> data = Tc2LtlabEvalConverter.convertRegressionModeId2Outcome(id2outcomeFile);
            double[] gold = new double[(int) data.size()];
            double[] prediction = new double[(int) data.size()];
            Iterator<EvaluationEntry<Double>> iterator = data.iterator();
            int i = 0;
            while (iterator.hasNext()) {
                EvaluationEntry<Double> next = iterator.next();
                gold[i] = next.getGold();
                prediction[i] = next.getPredicted();
                i++;
            }
            ScatterplotRenderer renderer = new ScatterplotRenderer(gold, prediction);
            getContext().storeBinary("scatterplot.pdf", renderer);
        }
    }
}

Example 4 with EvaluationData

use of de.unidue.ltl.evaluation.core.EvaluationData in project dkpro-tc by dkpro.

the class Tc2LtlabEvalConverter method convertSingleLabelModeId2Outcome.

/**
 * Loads a single-label DKPro TC id2outcome file into the evaluation data format
 *
 * @param id2OutcomeFile
 *            the id2outcome file
 * @return an evaluation data object
 * @throws Exception
 *             in case of error
 */
public static EvaluationData<String> convertSingleLabelModeId2Outcome(File id2OutcomeFile) throws Exception {
    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(id2OutcomeFile), "utf-8"));
    // pop first line
    reader.readLine();
    Map<String, String> map = buildMappingFromHeader(reader.readLine());
    EvaluationData<String> data = new EvaluationData<>();
    String line = null;
    while ((line = reader.readLine()) != null) {
        if (skipLine(line)) {
            continue;
        }
        int lastIdx = line.lastIndexOf("=");
        checkIndexRange(line, lastIdx);
        String docName = line.substring(0, lastIdx);
        String values = line.substring(lastIdx + 1);
        String[] valSplit = values.split(";");
        String prediction = map.get(valSplit[0]);
        String gold = map.get(valSplit[1]);
        // String threshold = valSplit[2];
        data.register(gold, prediction, docName);
    }
    reader.close();
    return data;
}

Example 5 with EvaluationData

use of de.unidue.ltl.evaluation.core.EvaluationData in project dkpro-tc by dkpro.

the class Tc2LtlabEvalConverter method convertMultiLabelModeId2Outcome.

/**
 * Loads a multi-label DKPro TC id2outcome file into the evaluation data format
 *
 * @param id2OutcomeFile
 *            the id2outcome file
 * @return an evaluation data object
 * @throws Exception
 *             in case of error
 */
public static EvaluationData<String> convertMultiLabelModeId2Outcome(File id2OutcomeFile) throws Exception {
    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(id2OutcomeFile), "utf-8"));
    // pop first line
    reader.readLine();
    Map<String, String> map = buildMappingFromHeader(reader.readLine());
    EvaluationData<String> data = new EvaluationData<>();
    String line = null;
    while ((line = reader.readLine()) != null) {
        if (skipLine(line)) {
            continue;
        }
        int lastIdx = line.lastIndexOf("=");
        checkIndexRange(line, lastIdx);
        String docName = line.substring(0, lastIdx);
        String values = line.substring(lastIdx + 1);
        String[] valSplit = values.split(";");
        Double threshold = Double.valueOf(valSplit[2]);
        String prediction = valSplit[0];
        List<String> mappedPred = convertMultiLabel(prediction.split(","), threshold, map);
        String gold = valSplit[1];
        List<String> mappedGold = convertMultiLabel(gold.split(","), threshold, map);
        data.registerMultiLabel(mappedGold, mappedPred, docName);
    }
    reader.close();
    return data;
}