Examples with Evaluation edu.cmu.minorthird.classify.experiments.Evaluation used on opensource projects

Example 1 with Evaluation

use of edu.cmu.minorthird.classify.experiments.Evaluation in project lucida by claritylab.

the class Experimenter method runExperiments.

/**
     * Runs the experiments specified in the input properties file.  The set of 
     * experiments run is the cross-product of the sets of algorithm configurations, training/testing
     * dataset configurations, and feature type combinations.
     * 
     */
public void runExperiments() {
    DecimalFormat format = new DecimalFormat("#0.00");
    HierarchicalClassifierTrainer qc = new HierarchicalClassifierTrainer(languagePair);
    StringBuilder sb = new StringBuilder();
    for (String alg : learningCombos) {
        properties.setProperty("learners", alg);
        for (String featureTypes : featureTypeCombos) {
            properties.setProperty("featureTypes", featureTypes);
            qc.setProperties(properties);
            Evaluation eval = qc.runExperiment();
            sb.append(alg.replaceAll(",", "-") + "-" + featureTypes.replaceAll(",", "-") + format.format((1.0 - eval.errorRate()) * 100) + "\n");
            String report = qc.createReport();
            log.debug("Report:\n" + report);
            FileUtil.writeFile(report, "reports/report-" + alg.replaceAll(",", "-") + "-" + featureTypes.replaceAll(",", "-") + ".txt", "UTF-8");
        }
    }
    FileUtil.writeFile(sb.toString(), "reports/results-" + System.currentTimeMillis() + ".txt", "UTF-8");
}

Example 2 with Evaluation

use of edu.cmu.minorthird.classify.experiments.Evaluation in project lucida by claritylab.

the class ScoreNormalizationFilter method evaluateAll.

/**
	 * Performs a cross-validation on the given data set for all combinations of
	 * features and models and writes a report for each evaluation. Determines
	 * the best combination according to the F1 measure.
	 * 
	 * @param serializedDir directory containing serialized results
	 * @param reportDir output directory for evaluation reports
	 * @return best combination of features and model
	 */
public static String[][] evaluateAll(String serializedDir, String reportDir) {
    // get all subsets of features
    Object[][] subsets = ArrayUtils.getNonemptySubsets(ALL_FEATURES);
    String[][] featureSets = new String[subsets.length][];
    for (int i = 0; i < subsets.length; i++) {
        featureSets[i] = new String[subsets[i].length];
        for (int j = 0; j < subsets[i].length; j++) featureSets[i][j] = (String) subsets[i][j];
    }
    //		// evaluate selected subsets of features only
    //		String[][] featureSets = {
    //			// all features
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MEAN_SCORE_F, MAX_SCORE_F, MIN_SCORE_F},
    //			// ignore extractors
    //			{SCORE_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MEAN_SCORE_F, MAX_SCORE_F, MIN_SCORE_F},
    //			// ignore answer type
    //			{SCORE_F, EXTRACTORS_F, NUM_ANSWERS_F, MEAN_SCORE_F, MAX_SCORE_F, MIN_SCORE_F},
    //			// ignore number of answers
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, MEAN_SCORE_F, MAX_SCORE_F, MIN_SCORE_F},
    //			// ignore mean score
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MAX_SCORE_F, MIN_SCORE_F},
    //			// ignore maximum score
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MEAN_SCORE_F, MIN_SCORE_F},
    //			// ignore minimum score
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MEAN_SCORE_F, MAX_SCORE_F},
    //			// ignore minimum score and maximum score
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F, NUM_ANSWERS_F, MEAN_SCORE_F},
    //			// only score, extractors and answer types
    //			{SCORE_F, EXTRACTORS_F, ANSWER_TYPES_F},
    //			// only score and extractors
    //			{SCORE_F, EXTRACTORS_F},
    //			// only score
    //			{SCORE_F}
    //		};
    //		// evaluate Ada Boost with different numbers of rounds
    //		int[] allNumBoosts = {10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200};
    // evaluate all combinations of features and models,
    // get best combination according to F1 measure
    double maxF1 = -1;
    String[][] bestCombination = new String[2][];
    for (String[] features : featureSets) //				String model = ADA_BOOST_N_M;
    for (String model : ALL_MODELS) {
        // do not overwrite an existing report
        String[] dataSets = FileUtils.getVisibleSubDirs(serializedDir);
        String filename = model + "_" + StringUtils.concat(features, "+") + "_" + StringUtils.concat(dataSets, "+");
        File reportFile = new File(reportDir, filename);
        if (reportFile.exists()) {
            MsgPrinter.printErrorMsg("File " + reportFile + " already exists.");
            continue;
        }
        // evaluate combination
        String msg = "Evaluating model " + model + " with feature(s) " + StringUtils.concat(features, ", ") + " (" + MsgPrinter.getTimestamp() + ")...";
        MsgPrinter.printStatusMsg(StringUtils.repeat("-", msg.length()));
        MsgPrinter.printStatusMsg(msg);
        MsgPrinter.printStatusMsg(StringUtils.repeat("-", msg.length()));
        long runTime = System.currentTimeMillis();
        Evaluation eval = evaluate(serializedDir, features, model);
        runTime = System.currentTimeMillis() - runTime;
        // write report
        String report = createReport(dataSets, features, model, eval, runTime);
        try {
            FileUtils.writeString(report, reportFile, "UTF-8");
        } catch (IOException e) {
            MsgPrinter.printErrorMsg("Failed to write report to file " + reportFile + ":");
            MsgPrinter.printErrorMsg(e.toString());
            System.exit(1);
        }
        // remember combination that yields highest F1 score
        double thisF1 = eval.f1();
        if (thisF1 > maxF1) {
            maxF1 = thisF1;
            bestCombination[0] = features;
            bestCombination[1] = new String[] { model };
        }
    }
    return bestCombination;
}

Example 3 with Evaluation

use of edu.cmu.minorthird.classify.experiments.Evaluation in project lucida by claritylab.

the class ScoreNormalizationFilter method evaluate.

/**
	 * Performs a cross-validation on the given data set for the given features
	 * and model.
	 * 
	 * @param serializedDir directory containing serialized results
	 * @param features selected features
	 * @param model selected model
	 * @return evaluation statistics
	 */
public static Evaluation evaluate(String serializedDir, String[] features, String model) {
    // create data set with selected features from serialized results
    Dataset dataSet = createDataset(features, serializedDir);
    // create learner for selected model
    ClassifierLearner learner = createLearner(model);
    // cross-validate model on data set
    RandomElement r = new RandomElement(System.currentTimeMillis());
    Splitter splitter = new CrossValSplitter(r, NUM_FOLDS);
    CrossValidatedDataset cvDataset = new CrossValidatedDataset(learner, dataSet, splitter, true);
    Evaluation eval = cvDataset.getEvaluation();
    return eval;
}

Example 4 with Evaluation

use of edu.cmu.minorthird.classify.experiments.Evaluation in project lucida by claritylab.

the class HierarchicalClassifierTrainer method main.

public static void main(String[] args) throws Exception {
    if (args.length > 3 || args.length < 2 || (args.length == 3 && !args[0].equals("--train"))) {
        System.err.println("Usage:");
        System.err.println("java HierarchicalClassifierTrainer [--train] <questionLang> <corpusLang>\n");
        System.err.println(" - <questionLang> and <corpusLang> must be one of the following:");
        System.err.println("     en_US, ja_JP, jp_JP, zh_TW, zh_CN");
        System.err.println(" - Outputs a trained model in the current directory if --train is used.");
        System.err.println(" - Otherwise, performs an evaluation using the configuration in the");
        System.err.println("     properties file and outputs a report describing the results.");
        System.exit(0);
    }
    boolean train = false;
    int langPairInd = 0;
    if (args[0].equals("--train")) {
        train = true;
        langPairInd++;
    }
    Pair<Language, Language> languagePair = new Pair<Language, Language>(Language.valueOf(args[langPairInd]), Language.valueOf(args[langPairInd + 1]));
    HierarchicalClassifierTrainer qct = new HierarchicalClassifierTrainer(languagePair);
    qct.initialize();
    if (train) {
        System.out.println("Training classifier...");
        qct.trainClassifier();
        qct.saveClassifier();
        System.out.println("Classifier saved.");
    } else {
        System.out.println("Running experiment...");
        Evaluation eval = qct.runExperiment();
        FileUtil.writeFile(qct.createReport(), args[0] + ".report" + System.currentTimeMillis() + ".txt", "UTF-8");
        ViewerFrame frame = new ViewerFrame(args[0], eval.toGUI());
        frame.setVisible(true);
    }
}