Examples with ClfDataSet edu.neu.ccs.pyramid.dataset.ClfDataSet used on opensource projects

Example 1 with ClfDataSet

use of edu.neu.ccs.pyramid.dataset.ClfDataSet in project pyramid by cheng-li.

the class GBClassifier method test.

private static void test(Config config) throws Exception {
    String output = config.getString("output.folder");
    File serializedModel = new File(output, "model");
    LKBoost lkBoost = (LKBoost) Serialization.deserialize(serializedModel);
    String sparsity = config.getString("input.matrixType");
    DataSetType dataSetType = null;
    switch(sparsity) {
        case "dense":
            dataSetType = DataSetType.CLF_DENSE;
            break;
        case "sparse":
            dataSetType = DataSetType.CLF_SPARSE;
            break;
        default:
            throw new IllegalArgumentException("input.matrixType should be dense or sparse");
    }
    ClfDataSet testSet = TRECFormat.loadClfDataSet(config.getString("input.testData"), dataSetType, true);
    System.out.println("test accuracy = " + Accuracy.accuracy(lkBoost, testSet));
    File reportFile = new File(output, "test_predictions.txt");
    report(lkBoost, testSet, reportFile);
    System.out.println("predictions on the test set are written to " + reportFile.getAbsolutePath());
    File probabilitiesFile = new File(output, "test_predicted_probabilities.txt");
    probabilities(lkBoost, testSet, probabilitiesFile);
    System.out.println("predicted probabilities on the test set are written to " + probabilitiesFile.getAbsolutePath());
}

Example 2 with ClfDataSet

use of edu.neu.ccs.pyramid.dataset.ClfDataSet in project pyramid by cheng-li.

the class L2BoostTest method buildTest.

static void buildTest() throws Exception {
    ClfDataSet dataSet = TRECFormat.loadClfDataSet(new File(DATASETS, "/spam/trec_data/train.trec"), DataSetType.CLF_SPARSE, true);
    System.out.println(dataSet.getMetaInfo());
    L2Boost boost = new L2Boost();
    RegTreeConfig regTreeConfig = new RegTreeConfig().setMaxNumLeaves(7);
    RegTreeFactory regTreeFactory = new RegTreeFactory(regTreeConfig);
    regTreeFactory.setLeafOutputCalculator(new L2BLeafOutputCalculator());
    L2BoostOptimizer optimizer = new L2BoostOptimizer(boost, dataSet, regTreeFactory);
    optimizer.setShrinkage(0.1);
    optimizer.initialize();
    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    for (int round = 0; round < 200; round++) {
        System.out.println("round=" + round);
        optimizer.iterate();
    }
    stopWatch.stop();
    System.out.println(stopWatch);
    double accuracy = Accuracy.accuracy(boost, dataSet);
    System.out.println("accuracy=" + accuracy);
    Serialization.serialize(boost, new File(TMP, "boost"));
}

Example 3 with ClfDataSet

use of edu.neu.ccs.pyramid.dataset.ClfDataSet in project pyramid by cheng-li.

the class L2BoostTest method test2.

static void test2() throws Exception {
    ClfDataSet dataSet = TRECFormat.loadClfDataSet(new File(DATASETS, "/spam/trec_data/train.trec"), DataSetType.CLF_SPARSE, true);
    System.out.println(dataSet.getMetaInfo());
    L2Boost boost = new L2Boost();
    L2BoostOptimizer optimizer = new L2BoostOptimizer(boost, dataSet);
    optimizer.setShrinkage(1);
    optimizer.initialize();
    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    for (int round = 0; round < 200; round++) {
        System.out.println("round=" + round);
        optimizer.iterate();
    }
    stopWatch.stop();
    System.out.println(stopWatch);
    double accuracy = Accuracy.accuracy(boost, dataSet);
    System.out.println("accuracy=" + accuracy);
}

Example 4 with ClfDataSet

use of edu.neu.ccs.pyramid.dataset.ClfDataSet in project pyramid by cheng-li.

the class GBClassifier method train.

private static void train(Config config) throws Exception {
    String sparsity = config.getString("input.matrixType");
    DataSetType dataSetType = null;
    switch(sparsity) {
        case "dense":
            dataSetType = DataSetType.CLF_DENSE;
            break;
        case "sparse":
            dataSetType = DataSetType.CLF_SPARSE;
            break;
        default:
            throw new IllegalArgumentException("input.matrixType should be dense or sparse");
    }
    ClfDataSet trainSet = TRECFormat.loadClfDataSet(config.getString("input.trainData"), dataSetType, true);
    ClfDataSet testSet = null;
    if (config.getBoolean("train.showTestProgress")) {
        testSet = TRECFormat.loadClfDataSet(config.getString("input.testData"), dataSetType, true);
    }
    int numClasses = trainSet.getNumClasses();
    LKBoost lkBoost = new LKBoost(numClasses);
    RegTreeConfig regTreeConfig = new RegTreeConfig().setMaxNumLeaves(config.getInt("train.numLeaves"));
    RegTreeFactory regTreeFactory = new RegTreeFactory(regTreeConfig);
    regTreeFactory.setLeafOutputCalculator(new LKBOutputCalculator(numClasses));
    LKBoostOptimizer optimizer = new LKBoostOptimizer(lkBoost, trainSet, regTreeFactory);
    optimizer.setShrinkage(config.getDouble("train.shrinkage"));
    optimizer.initialize();
    int progressInterval = config.getInt("train.showProgress.interval");
    int numIterations = config.getInt("train.numIterations");
    for (int i = 1; i <= numIterations; i++) {
        System.out.println("iteration " + i);
        optimizer.iterate();
        if (config.getBoolean("train.showTrainProgress") && (i % progressInterval == 0 || i == numIterations)) {
            System.out.println("training accuracy = " + Accuracy.accuracy(lkBoost, trainSet));
        }
        if (config.getBoolean("train.showTestProgress") && (i % progressInterval == 0 || i == numIterations)) {
            System.out.println("test accuracy = " + Accuracy.accuracy(lkBoost, testSet));
        }
    }
    System.out.println("training done!");
    String output = config.getString("output.folder");
    new File(output).mkdirs();
    File serializedModel = new File(output, "model");
    Serialization.serialize(lkBoost, serializedModel);
    System.out.println("model saved to " + serializedModel.getAbsolutePath());
    File reportFile = new File(output, "train_predictions.txt");
    report(lkBoost, trainSet, reportFile);
    System.out.println("predictions on the training set are written to " + reportFile.getAbsolutePath());
    File probabilitiesFile = new File(output, "train_predicted_probabilities.txt");
    probabilities(lkBoost, trainSet, probabilitiesFile);
    System.out.println("predicted probabilities on the training set are written to " + probabilitiesFile.getAbsolutePath());
}

Example 5 with ClfDataSet

use of edu.neu.ccs.pyramid.dataset.ClfDataSet in project pyramid by cheng-li.

the class PlattScaling method fitClassK.

private static LogisticRegression fitClassK(double[] scores, int[] labels) {
    ClfDataSet dataSet = ClfDataSetBuilder.getBuilder().numClasses(2).numDataPoints(scores.length).numFeatures(1).dense(true).missingValue(false).build();
    for (int i = 0; i < scores.length; i++) {
        dataSet.setFeatureValue(i, 0, scores[i]);
        dataSet.setLabel(i, labels[i]);
    }
    LogisticRegression logisticRegression = new LogisticRegression(2, dataSet.getNumFeatures());
    ElasticNetLogisticTrainer trainer = ElasticNetLogisticTrainer.newBuilder(logisticRegression, dataSet).setRegularization(1.0E-9).setL1Ratio(0).build();
    trainer.optimize();
    return logisticRegression;
}