Examples with ModelMetricsBinomial hex.ModelMetricsBinomial used on opensource projects

Example 6 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testRowWeightsOne.

@Test
public void testRowWeightsOne() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/weights_all_ones.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._weights_column = "weight";
        parms._seed = 234;
        parms._min_rows = 1;
        parms._max_depth = 2;
        parms._ntrees = 3;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        // OOB
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._training_metrics;
        assertEquals(_AUC, mm.auc_obj()._auc, 1e-8);
        assertEquals(_MSE, mm.mse(), 1e-8);
        assertEquals(_LogLoss, mm.logloss(), 1e-6);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null)
            drf.delete();
        Scope.exit();
    }
}

Example 7 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testNfoldsOneVsRest.

@Test
public void testNfoldsOneVsRest() {
    Frame tfr = null;
    DRFModel drf1 = null;
    DRFModel drf2 = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/weights.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._seed = 9999;
        parms._min_rows = 2;
        parms._nfolds = (int) tfr.numRows();
        parms._fold_assignment = Model.Parameters.FoldAssignmentScheme.Modulo;
        parms._max_depth = 5;
        parms._ntrees = 5;
        drf1 = new DRF(parms).trainModel().get();
        //            parms._nfolds = (int) tfr.numRows() + 1; //this is now an error
        drf2 = new DRF(parms).trainModel().get();
        ModelMetricsBinomial mm1 = (ModelMetricsBinomial) drf1._output._cross_validation_metrics;
        ModelMetricsBinomial mm2 = (ModelMetricsBinomial) drf2._output._cross_validation_metrics;
        assertEquals(mm1.auc_obj()._auc, mm2.auc_obj()._auc, 1e-12);
        assertEquals(mm1.mse(), mm2.mse(), 1e-12);
        assertEquals(mm1.logloss(), mm2.logloss(), 1e-12);
    //TODO: add check: the correct number of individual models were built. PUBDEV-1690
    } finally {
        if (tfr != null)
            tfr.remove();
        if (drf1 != null) {
            drf1.deleteCrossValidationModels();
            drf1.delete();
        }
        if (drf2 != null) {
            drf2.deleteCrossValidationModels();
            drf2.delete();
        }
        Scope.exit();
    }
}

Example 8 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testRowWeightsTwo.

@Test
public void testRowWeightsTwo() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("smalldata/junit/weights_all_twos.csv");
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "response";
        parms._weights_column = "weight";
        parms._seed = 234;
        //in terms of weighted rows
        parms._min_rows = 2;
        parms._max_depth = 2;
        parms._ntrees = 3;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        // OOB
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._training_metrics;
        assertEquals(_AUC, mm.auc_obj()._auc, 1e-8);
        assertEquals(_MSE, mm.mse(), 1e-8);
        assertEquals(_LogLoss, mm.logloss(), 1e-6);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null)
            drf.delete();
        Scope.exit();
    }
}

Example 9 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DRFTest method testNFold.

@Ignore
@Test
public void testNFold() {
    Frame tfr = null, vfr = null;
    DRFModel drf = null;
    Scope.enter();
    try {
        tfr = parse_test_file("./smalldata/airlines/allyears2k_headers.zip");
        for (String s : new String[] { "DepTime", "ArrTime", "ActualElapsedTime", "AirTime", "ArrDelay", "DepDelay", "Cancelled", "CancellationCode", "CarrierDelay", "WeatherDelay", "NASDelay", "SecurityDelay", "LateAircraftDelay", "IsArrDelayed" }) {
            tfr.remove(s).remove();
        }
        DKV.put(tfr);
        DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
        parms._train = tfr._key;
        parms._response_column = "IsDepDelayed";
        parms._seed = 234;
        parms._min_rows = 2;
        parms._nfolds = 3;
        parms._max_depth = 5;
        parms._ntrees = 5;
        // Build a first model; all remaining models should be equal
        drf = new DRF(parms).trainModel().get();
        ModelMetricsBinomial mm = (ModelMetricsBinomial) drf._output._cross_validation_metrics;
        // 1 node
        assertEquals(0.7276154565296726, mm.auc_obj()._auc, 1e-8);
        assertEquals(0.21211607823987555, mm.mse(), 1e-8);
        assertEquals(0.6121968624307211, mm.logloss(), 1e-6);
    } finally {
        if (tfr != null)
            tfr.remove();
        if (vfr != null)
            vfr.remove();
        if (drf != null) {
            drf.deleteCrossValidationModels();
            drf.delete();
        }
        Scope.exit();
    }
}

Example 10 with ModelMetricsBinomial

use of hex.ModelMetricsBinomial in project h2o-3 by h2oai.

the class DeepWaterAbstractIntegrationTest method MOJOTest.

private void MOJOTest(Model.Parameters.CategoricalEncodingScheme categoricalEncodingScheme, boolean enumCols, boolean standardize) {
    Frame tr = null;
    Frame tr2 = null;
    Frame tr3 = null;
    DeepWaterModel m = null;
    Frame preds = null;
    Frame preds2 = null;
    Frame preds3 = null;
    try {
        DeepWaterParameters p = new DeepWaterParameters();
        tr = parse_test_file("smalldata/prostate/prostate.csv");
        p._response_column = "CAPSULE";
        for (String col : new String[] { p._response_column }) {
            Vec v = tr.remove(col);
            tr.add(col, v.toCategoricalVec());
            v.remove();
        }
        if (enumCols) {
            for (String col : new String[] { "RACE", "DPROS", "DCAPS", "GLEASON" }) {
                Vec v = tr.remove(col);
                tr.add(col, v.toCategoricalVec());
                v.remove();
            }
        }
        DKV.put(tr);
        p._train = tr._key;
        p._ignored_columns = new String[] { "ID" };
        p._backend = getBackend();
        p._seed = 12345;
        p._epochs = 5;
        p._categorical_encoding = categoricalEncodingScheme;
        p._standardize = standardize;
        p._hidden = new int[] { 50, 50 };
        m = new DeepWater(p).trainModel().get();
        // Score original training frame
        preds = m.score(tr);
        Assert.assertTrue(m.testJavaScoring(tr, preds, 1e-3));
        double auc = ModelMetricsBinomial.make(preds.vec(2), tr.vec(p._response_column)).auc();
        Assert.assertTrue(Math.abs(auc - ((ModelMetricsBinomial) m._output._training_metrics).auc()) < 1e-3);
        if (standardize)
            Assert.assertTrue(auc > 0.7);
        // Score all numeric frame (cols in the right order) - do the transformation to enum on the fly
        tr2 = parse_test_file("smalldata/prostate/prostate.csv");
        for (String col : new String[] { p._response_column }) {
            //DO NOT CONVERT TO ENUM
            tr2.add(col, tr2.remove(col));
        }
        if (enumCols) {
            for (String col : new String[] { "RACE", "DPROS", "DCAPS", "GLEASON" }) {
                //DO NOT CONVERT TO ENUM
                tr2.add(col, tr2.remove(col));
            }
        }
        preds2 = m.score(tr2);
        auc = ModelMetricsBinomial.make(preds2.vec(2), tr2.vec(p._response_column)).auc();
        Assert.assertTrue(Math.abs(auc - ((ModelMetricsBinomial) m._output._training_metrics).auc()) < 1e-3);
        if (standardize)
            Assert.assertTrue(auc > 0.7);
        // Score all numeric frame (cols in the wrong order) - do the transformation to enum on the fly
        tr3 = parse_test_file("smalldata/prostate/prostate.csv");
        preds3 = m.score(tr3);
        auc = ModelMetricsBinomial.make(preds3.vec(2), tr3.vec(p._response_column)).auc();
        Assert.assertTrue(Math.abs(auc - ((ModelMetricsBinomial) m._output._training_metrics).auc()) < 1e-3);
        if (standardize)
            Assert.assertTrue(auc > 0.7);
    } finally {
        if (tr != null)
            tr.remove();
        if (tr2 != null)
            tr2.remove();
        if (tr3 != null)
            tr3.remove();
        if (m != null)
            m.remove();
        if (preds != null)
            preds.remove();
        if (preds2 != null)
            preds2.remove();
        if (preds3 != null)
            preds3.remove();
    }
}