Search in sources :

Example 1 with AUC

use of water.api.AUC in project h2o-2 by h2oai.

the class DeepLearningProstateTest method runFraction.

public void runFraction(float fraction) {
    long seed = 0xDECAF;
    Random rng = new Random(seed);
    String[] datasets = new String[2];
    int[][] responses = new int[datasets.length][];
    //CAPSULE (binomial), AGE (regression), GLEASON (multi-class)
    datasets[0] = "smalldata/./logreg/prostate.csv";
    //CAPSULE (binomial), AGE (regression), GLEASON (multi-class)
    responses[0] = new int[] { 1, 2, 8 };
    //Iris-type (multi-class)
    datasets[1] = "smalldata/iris/iris.csv";
    //Iris-type (multi-class)
    responses[1] = new int[] { 4 };
    int testcount = 0;
    int count = 0;
    for (int i = 0; i < datasets.length; ++i) {
        String dataset = datasets[i];
        Key file = NFSFileVec.make(find_test_file(dataset));
        Frame frame = ParseDataset2.parse(Key.make(), new Key[] { file });
        Key vfile = NFSFileVec.make(find_test_file(dataset));
        Frame vframe = ParseDataset2.parse(Key.make(), new Key[] { vfile });
        try {
            for (boolean replicate : new boolean[] { true, false }) {
                for (boolean load_balance : new boolean[] { true, false }) {
                    for (boolean shuffle : new boolean[] { true, false }) {
                        for (boolean balance_classes : new boolean[] { true, false }) {
                            for (int resp : responses[i]) {
                                for (DeepLearning.ClassSamplingMethod csm : new DeepLearning.ClassSamplingMethod[] { DeepLearning.ClassSamplingMethod.Stratified, DeepLearning.ClassSamplingMethod.Uniform }) {
                                    for (int scoretraining : new int[] { 200, 20, 0 }) {
                                        for (int scorevalidation : new int[] { 200, 20, 0 }) {
                                            for (int vf : new int[] { //no validation
                                            0, //same as source
                                            1, //different validation frame
                                            -1 }) {
                                                for (int n_folds : new int[] { 0, 2 }) {
                                                    if (n_folds != 0 && vf != 0)
                                                        continue;
                                                    for (boolean keep_cv_splits : new boolean[] { false }) {
                                                        //otherwise it leaks
                                                        for (boolean override_with_best_model : new boolean[] { false, true }) {
                                                            for (int train_samples_per_iteration : new int[] { //auto-tune
                                                            -2, //N epochs per iteration
                                                            -1, //1 epoch per iteration
                                                            0, // <1 epoch per iteration
                                                            rng.nextInt(200), //>1 epoch per iteration
                                                            500 }) {
                                                                DeepLearningModel model1 = null, model2 = null;
                                                                Key dest = null, dest_tmp = null;
                                                                count++;
                                                                if (fraction < rng.nextFloat())
                                                                    continue;
                                                                try {
                                                                    Log.info("**************************)");
                                                                    Log.info("Starting test #" + count);
                                                                    Log.info("**************************)");
                                                                    final double epochs = 7 + rng.nextDouble() + rng.nextInt(4);
                                                                    final int[] hidden = new int[] { 1 + rng.nextInt(4), 1 + rng.nextInt(6) };
                                                                    //no validation
                                                                    Frame valid = null;
                                                                    if (//use the same frame for validation
                                                                    vf == 1)
                                                                        //use the same frame for validation
                                                                        valid = frame;
                                                                    else //different validation frame (here: from the same file)
                                                                    if (vf == -1)
                                                                        valid = vframe;
                                                                    // build the model, with all kinds of shuffling/rebalancing/sampling
                                                                    dest_tmp = Key.make("first");
                                                                    {
                                                                        Log.info("Using seed: " + seed);
                                                                        DeepLearning p = new DeepLearning();
                                                                        p.checkpoint = null;
                                                                        p.destination_key = dest_tmp;
                                                                        p.source = frame;
                                                                        p.response = frame.vecs()[resp];
                                                                        p.validation = valid;
                                                                        p.hidden = hidden;
                                                                        if (i == 0 && resp == 2)
                                                                            p.classification = false;
                                                                        //                                      p.best_model_key = best_model_key;
                                                                        p.override_with_best_model = override_with_best_model;
                                                                        p.epochs = epochs;
                                                                        p.n_folds = n_folds;
                                                                        p.keep_cross_validation_splits = keep_cv_splits;
                                                                        p.seed = seed;
                                                                        p.train_samples_per_iteration = train_samples_per_iteration;
                                                                        p.force_load_balance = load_balance;
                                                                        p.replicate_training_data = replicate;
                                                                        p.shuffle_training_data = shuffle;
                                                                        p.score_training_samples = scoretraining;
                                                                        p.score_validation_samples = scorevalidation;
                                                                        p.classification_stop = -1;
                                                                        p.regression_stop = -1;
                                                                        p.balance_classes = balance_classes;
                                                                        p.quiet_mode = true;
                                                                        p.score_validation_sampling = csm;
                                                                        try {
                                                                            p.invoke();
                                                                        } catch (Throwable t) {
                                                                            t.printStackTrace();
                                                                            throw new RuntimeException(t);
                                                                        } finally {
                                                                            p.delete();
                                                                        }
                                                                        model1 = UKV.get(dest_tmp);
                                                                        assert (((p.train_samples_per_iteration <= 0 || p.train_samples_per_iteration >= frame.numRows()) && model1.epoch_counter > epochs) || Math.abs(model1.epoch_counter - epochs) / epochs < 0.20);
                                                                        if (n_folds != 0) // test HTML of cv models
                                                                        {
                                                                            for (Key k : model1.get_params().xval_models) {
                                                                                DeepLearningModel cv_model = UKV.get(k);
                                                                                StringBuilder sb = new StringBuilder();
                                                                                cv_model.generateHTML("cv", sb);
                                                                                cv_model.delete_best_model();
                                                                                cv_model.delete();
                                                                            }
                                                                        }
                                                                    }
                                                                    // Do some more training via checkpoint restart
                                                                    // For n_folds, continue without n_folds (not yet implemented) - from now on, model2 will have n_folds=0...
                                                                    dest = Key.make("restart");
                                                                    DeepLearning p = new DeepLearning();
                                                                    //this actually *requires* frame to also still be in UKV (because of DataInfo...)
                                                                    final DeepLearningModel tmp_model = UKV.get(dest_tmp);
                                                                    //HEX-1817
                                                                    Assert.assertTrue(tmp_model.get_params().state == Job.JobState.DONE);
                                                                    Assert.assertTrue(tmp_model.model_info().get_processed_total() >= frame.numRows() * epochs);
                                                                    assert (tmp_model != null);
                                                                    p.checkpoint = dest_tmp;
                                                                    p.destination_key = dest;
                                                                    p.n_folds = 0;
                                                                    p.source = frame;
                                                                    p.validation = valid;
                                                                    p.response = frame.vecs()[resp];
                                                                    if (i == 0 && resp == 2)
                                                                        p.classification = false;
                                                                    p.override_with_best_model = override_with_best_model;
                                                                    p.epochs = epochs;
                                                                    p.seed = seed;
                                                                    p.train_samples_per_iteration = train_samples_per_iteration;
                                                                    try {
                                                                        p.invoke();
                                                                    } catch (Throwable t) {
                                                                        t.printStackTrace();
                                                                        throw new RuntimeException(t);
                                                                    } finally {
                                                                        p.delete();
                                                                    }
                                                                    // score and check result (on full data)
                                                                    //this actually *requires* frame to also still be in UKV (because of DataInfo...)
                                                                    model2 = UKV.get(dest);
                                                                    //HEX-1817
                                                                    Assert.assertTrue(model2.get_params().state == Job.JobState.DONE);
                                                                    // test HTML
                                                                    {
                                                                        StringBuilder sb = new StringBuilder();
                                                                        model2.generateHTML("test", sb);
                                                                    }
                                                                    // score and check result of the best_model
                                                                    if (model2.actual_best_model_key != null) {
                                                                        final DeepLearningModel best_model = UKV.get(model2.actual_best_model_key);
                                                                        //HEX-1817
                                                                        Assert.assertTrue(best_model.get_params().state == Job.JobState.DONE);
                                                                        // test HTML
                                                                        {
                                                                            StringBuilder sb = new StringBuilder();
                                                                            best_model.generateHTML("test", sb);
                                                                        }
                                                                        if (override_with_best_model) {
                                                                            Assert.assertEquals(best_model.error(), model2.error(), 0);
                                                                        }
                                                                    }
                                                                    if (valid == null)
                                                                        valid = frame;
                                                                    double threshold = 0;
                                                                    if (model2.isClassifier()) {
                                                                        Frame pred = null, pred2 = null;
                                                                        try {
                                                                            pred = model2.score(valid);
                                                                            StringBuilder sb = new StringBuilder();
                                                                            AUC auc = new AUC();
                                                                            double error = 0;
                                                                            // binary
                                                                            if (model2.nclasses() == 2) {
                                                                                auc.actual = valid;
                                                                                assert (resp == 1);
                                                                                auc.vactual = valid.vecs()[resp];
                                                                                auc.predict = pred;
                                                                                auc.vpredict = pred.vecs()[2];
                                                                                auc.invoke();
                                                                                auc.toASCII(sb);
                                                                                AUCData aucd = auc.data();
                                                                                threshold = aucd.threshold();
                                                                                error = aucd.err();
                                                                                Log.info(sb);
                                                                                // check that auc.cm() is the right CM
                                                                                Assert.assertEquals(new ConfusionMatrix(aucd.cm()).err(), error, 1e-15);
                                                                                // check that calcError() is consistent as well (for CM=null, AUC!=null)
                                                                                Assert.assertEquals(model2.calcError(valid, auc.vactual, pred, pred, "training", false, 0, null, auc, null), error, 1e-15);
                                                                            }
                                                                            // Compute CM
                                                                            double CMerrorOrig;
                                                                            {
                                                                                sb = new StringBuilder();
                                                                                water.api.ConfusionMatrix CM = new water.api.ConfusionMatrix();
                                                                                CM.actual = valid;
                                                                                CM.vactual = valid.vecs()[resp];
                                                                                CM.predict = pred;
                                                                                CM.vpredict = pred.vecs()[0];
                                                                                CM.invoke();
                                                                                sb.append("\n");
                                                                                sb.append("Threshold: " + "default\n");
                                                                                CM.toASCII(sb);
                                                                                Log.info(sb);
                                                                                CMerrorOrig = new ConfusionMatrix(CM.cm).err();
                                                                            }
                                                                            // confirm that orig CM was made with threshold 0.5
                                                                            // put pred2 into UKV, and allow access
                                                                            pred2 = new Frame(Key.make("pred2"), pred.names(), pred.vecs());
                                                                            pred2.delete_and_lock(null);
                                                                            pred2.unlock(null);
                                                                            if (model2.nclasses() == 2) {
                                                                                // make labels with 0.5 threshold for binary classifier
                                                                                Env ev = Exec2.exec("pred2[,1]=pred2[,3]>=" + 0.5);
                                                                                try {
                                                                                    pred2 = ev.popAry();
                                                                                    String skey = ev.key();
                                                                                    ev.subRef(pred2, skey);
                                                                                } finally {
                                                                                    if (ev != null)
                                                                                        ev.remove_and_unlock();
                                                                                }
                                                                                water.api.ConfusionMatrix CM = new water.api.ConfusionMatrix();
                                                                                CM.actual = valid;
                                                                                CM.vactual = valid.vecs()[1];
                                                                                CM.predict = pred2;
                                                                                CM.vpredict = pred2.vecs()[0];
                                                                                CM.invoke();
                                                                                sb = new StringBuilder();
                                                                                sb.append("\n");
                                                                                sb.append("Threshold: " + 0.5 + "\n");
                                                                                CM.toASCII(sb);
                                                                                Log.info(sb);
                                                                                double threshErr = new ConfusionMatrix(CM.cm).err();
                                                                                Assert.assertEquals(threshErr, CMerrorOrig, 1e-15);
                                                                                // make labels with AUC-given threshold for best F1
                                                                                ev = Exec2.exec("pred2[,1]=pred2[,3]>=" + threshold);
                                                                                try {
                                                                                    pred2 = ev.popAry();
                                                                                    String skey = ev.key();
                                                                                    ev.subRef(pred2, skey);
                                                                                } finally {
                                                                                    if (ev != null)
                                                                                        ev.remove_and_unlock();
                                                                                }
                                                                                CM = new water.api.ConfusionMatrix();
                                                                                CM.actual = valid;
                                                                                CM.vactual = valid.vecs()[1];
                                                                                CM.predict = pred2;
                                                                                CM.vpredict = pred2.vecs()[0];
                                                                                CM.invoke();
                                                                                sb = new StringBuilder();
                                                                                sb.append("\n");
                                                                                sb.append("Threshold: ").append(threshold).append("\n");
                                                                                CM.toASCII(sb);
                                                                                Log.info(sb);
                                                                                double threshErr2 = new ConfusionMatrix(CM.cm).err();
                                                                                Assert.assertEquals(threshErr2, error, 1e-15);
                                                                            }
                                                                        } finally {
                                                                            if (pred != null)
                                                                                pred.delete();
                                                                            if (pred2 != null)
                                                                                pred2.delete();
                                                                        }
                                                                    }
                                                                    //classifier
                                                                    Log.info("Parameters combination " + count + ": PASS");
                                                                    testcount++;
                                                                } catch (Throwable t) {
                                                                    t.printStackTrace();
                                                                    throw new RuntimeException(t);
                                                                } finally {
                                                                    if (model1 != null) {
                                                                        model1.delete_xval_models();
                                                                        model1.delete_best_model();
                                                                        model1.delete();
                                                                    }
                                                                    if (model2 != null) {
                                                                        model2.delete_xval_models();
                                                                        model2.delete_best_model();
                                                                        model2.delete();
                                                                    }
                                                                }
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } finally {
            frame.delete();
            vframe.delete();
        }
    }
    Log.info("\n\n=============================================");
    Log.info("Tested " + testcount + " out of " + count + " parameter combinations.");
    Log.info("=============================================");
}
Also used : Frame(water.fvec.Frame) AUCData(water.api.AUCData) DeepLearning(hex.deeplearning.DeepLearning) Env(water.exec.Env) AUC(water.api.AUC) Random(java.util.Random) water(water) DeepLearningModel(hex.deeplearning.DeepLearningModel)

Example 2 with AUC

use of water.api.AUC in project h2o-2 by h2oai.

the class GLMValidation method computeAUC.

protected void computeAUC() {
    if (_glm.family == Family.binomial) {
        for (ConfusionMatrix cm : _cms) cm.reComputeErrors();
        AUC auc = new AUC(_cms, thresholds, /*TODO: add CM domain*/
        null);
        this.auc = auc.data().AUC();
        best_threshold = auc.data().threshold();
    }
}
Also used : ConfusionMatrix(hex.ConfusionMatrix) AUC(water.api.AUC)

Aggregations

AUC (water.api.AUC)2 ConfusionMatrix (hex.ConfusionMatrix)1 DeepLearning (hex.deeplearning.DeepLearning)1 DeepLearningModel (hex.deeplearning.DeepLearningModel)1 Random (java.util.Random)1 water (water)1 AUCData (water.api.AUCData)1 Env (water.exec.Env)1 Frame (water.fvec.Frame)1