Examples with ModelMetricsBinomial hex.ModelMetricsBinomial used on opensource projects

Example 1 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testRowWeightsTiny.

@Test
public void testRowWeightsTiny() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/weights_all_tiny.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._weights_column = "weight";
        parms._seed = 234;
        // in terms of weighted rows
        parms._min_rows = 0.01242;
        parms._max_depth = 2;
        parms._ntrees = 3;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        // OOB
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._training_metrics;
        assertEquals(_AUC, mm.auc_obj()._auc, 1e-8);
        assertEquals(_MSE, mm.mse(), 1e-8);
        assertEquals(_LogLoss, mm.logloss(), 1e-6);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null)
            drf.delete();
        Scope.exit();
    }
}

Example 2 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testRowWeights.

@Test
public void testRowWeights() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/weights.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._weights_column = "weight";
        parms._seed = 234;
        parms._min_rows = 1;
        parms._max_depth = 2;
        parms._ntrees = 3;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        // OOB
        // Reduced number of rows changes the row sampling -> results differ
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._training_metrics;
        assertEquals(1.0, mm.auc_obj()._auc, 1e-8);
        assertEquals(0.05823863636363636, mm.mse(), 1e-8);
        assertEquals(0.21035264541934587, mm.logloss(), 1e-6);
        // test set scoring (on the same dataset, but without normalizing the weights)
        Frame pred = drf.score(parms.train());
        hex.ModelMetricsBinomial mm2 = hex.ModelMetricsBinomial.getFromDKV(drf, parms.train());
        // Non-OOB
        assertEquals(1, mm2.auc_obj()._auc, 1e-8);
        assertEquals(0.0154320987654321, mm2.mse(), 1e-8);
        assertEquals(0.08349430638608361, mm2.logloss(), 1e-8);
        pred.remove();
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null)
            drf.delete();
        Scope.exit();
    }
}

Example 3 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testNoRowWeightsShuffled.

@Test
public void testNoRowWeightsShuffled() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/no_weights_shuffled.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._seed = 234;
        parms._min_rows = 1;
        parms._max_depth = 2;
        parms._ntrees = 3;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        // OOB
        // Shuffling changes the row sampling -> results differ
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._training_metrics;
        assertEquals(1.0, mm.auc_obj()._auc, 1e-8);
        assertEquals(0.0290178571428571443, mm.mse(), 1e-8);
        assertEquals(0.10824081452821664, mm.logloss(), 1e-6);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null)
            drf.delete();
        Scope.exit();
    }
}

Example 4 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testNfoldsConsecutiveModelsSame.

@Test
public void testNfoldsConsecutiveModelsSame() {
    Frame tfr = null;
    Vec old = null;
    DRFModel drf1 = null;
    DRFModel drf2 = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/cars_20mpg.csv");
        // Remove unique id
        tfr.remove("name").remove();
        tfr.remove("economy").remove();
        old = tfr.remove("economy_20mpg");
        // response to last column
        tfr.add("economy_20mpg", VecUtils.toCategoricalVec(old));
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "economy_20mpg";
        parms._min_rows = 2;
        parms._max_depth = 2;
        parms._nfolds = 3;
        parms._ntrees = 3;
        parms._seed = 77777;
        drf1 = new DRF(parms).trainModel().get();
        drf2 = new DRF(parms).trainModel().get();
        ModelMetricsBinomial mm1 = (ModelMetricsBinomial) drf1._output._cross_validation_metrics;
        ModelMetricsBinomial mm2 = (ModelMetricsBinomial) drf2._output._cross_validation_metrics;
        assertEquals(mm1.auc_obj()._auc, mm2.auc_obj()._auc, 1e-12);
        assertEquals(mm1.mse(), mm2.mse(), 1e-12);
        assertEquals(mm1.logloss(), mm2.logloss(), 1e-12);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (old != null)
            old.remove();
        if (drf1 != null) {
            drf1.deleteCrossValidationModels();
            drf1.delete();
        }
        if (drf2 != null) {
            drf2.deleteCrossValidationModels();
            drf2.delete();
        }
        Scope.exit();
    }
}

Example 5 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DeepLearningSpiralsTest method run.

@Test
public void run() {
    Scope.enter();
    NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/junit/two_spiral.csv");
    Frame frame = ParseDataset.parse(Key.make(), nfs._key);
    Log.info(frame);
    int resp = frame.names().length - 1;
    for (boolean sparse : new boolean[] { true, false }) {
        for (boolean col_major : new boolean[] { false }) {
            if (!sparse && col_major)
                continue;
            Key model_id = Key.make();
            // build the model
            {
                DeepLearningParameters p = new DeepLearningParameters();
                p._epochs = 5000;
                p._hidden = new int[] { 100 };
                p._sparse = sparse;
                p._col_major = col_major;
                p._activation = DeepLearningParameters.Activation.Tanh;
                p._initial_weight_distribution = DeepLearningParameters.InitialWeightDistribution.Normal;
                p._initial_weight_scale = 2.5;
                p._loss = DeepLearningParameters.Loss.CrossEntropy;
                p._train = frame._key;
                p._response_column = frame.names()[resp];
                // Convert response to categorical
                Scope.track(frame.replace(resp, frame.vecs()[resp].toCategoricalVec()));
                DKV.put(frame);
                p._rho = 0.99;
                p._epsilon = 5e-3;
                //stop when reaching 0 classification error on training data
                p._classification_stop = 0;
                p._train_samples_per_iteration = 10000;
                p._stopping_rounds = 5;
                p._stopping_metric = ScoreKeeper.StoppingMetric.misclassification;
                p._score_each_iteration = true;
                p._reproducible = true;
                p._seed = 1234;
                new DeepLearning(p, model_id).trainModel().get();
            }
            // score and check result
            {
                DeepLearningModel mymodel = DKV.getGet(model_id);
                Frame pred = mymodel.score(frame);
                ModelMetricsBinomial mm = ModelMetricsBinomial.getFromDKV(mymodel, frame);
                double error = mm._auc.defaultErr();
                Log.info("Error: " + error);
                if (error > 0.1) {
                    Assert.fail("Test classification error is not <= 0.1, but " + error + ".");
                }
                Assert.assertTrue(mymodel.testJavaScoring(frame, pred, 1e-6));
                pred.delete();
                mymodel.delete();
            }
        }
    }
    frame.delete();
    Scope.exit();
}