Example 1 with ConfusionMatrix
use of hex.ConfusionMatrix in project h2o-2 by h2oai.
the class SpeeDRFModel method scoreOnTest.
private void scoreOnTest(Frame fr, Vec modelResp) {
Frame scored = score(fr);
water.api.ConfusionMatrix cm = new water.api.ConfusionMatrix();
cm.vactual = fr.lastVec();
cm.vpredict = scored.anyVec();
cm.invoke();
// Regression scoring
if (regression) {
float mse = (float) cm.mse;
errs[errs.length - 1] = mse;
cms[cms.length - 1] = null;
// Classification scoring
} else {
Vec lv = scored.lastVec();
double mse = CMTask.MSETask.doTask(scored.add("actual", fr.lastVec()));
this.cm = cm.cm;
errs[errs.length - 1] = (float) mse;
ConfusionMatrix new_cm = new ConfusionMatrix(this.cm);
cms[cms.length - 1] = new_cm;
// Create the ROC Plot
if (classes() == 2) {
Vec v = null;
Frame fa = null;
if (lv.isInt()) {
fa = new MRTask2() {
@Override
public void map(Chunk[] cs, NewChunk nchk) {
int rows = cs[0]._len;
int cols = cs.length - 1;
for (int r = 0; r < rows; ++r) {
nchk.addNum(cs[cols].at0(r) == 0 ? 1e-10 : 1.0 - 1e-10);
}
}
}.doAll(1, scored).outputFrame(null, null);
v = fa.anyVec();
}
AUC auc_calc = new AUC();
auc_calc.vactual = cm.vactual;
// lastVec is class1
auc_calc.vpredict = v == null ? lv : v;
auc_calc.invoke();
validAUC = auc_calc.data();
if (v != null)
UKV.remove(v._key);
if (fa != null)
fa.delete();
UKV.remove(lv._key);
}
}
scored.remove("actual");
scored.delete();
}Example 2 with ConfusionMatrix
use of hex.ConfusionMatrix in project h2o-2 by h2oai.
the class GLMValidation method computeAUC.
protected void computeAUC() {
if (_glm.family == Family.binomial) {
for (ConfusionMatrix cm : _cms) cm.reComputeErrors();
AUC auc = new AUC(_cms, thresholds, /*TODO: add CM domain*/
null);
this.auc = auc.data().AUC();
best_threshold = auc.data().threshold();
}
}
Also used :
ConfusionMatrix(hex.ConfusionMatrix)
AUC(water.api.AUC)
Example 3 with ConfusionMatrix
use of hex.ConfusionMatrix in project h2o-3 by h2oai.
the class DeepLearningProstateTest method runFraction.
public void runFraction(float fraction) {
long seed = 0xDECAFFF;
Random rng = new Random(seed);
String[] datasets = new String[2];
int[][] responses = new int[datasets.length][];
//CAPSULE (binomial), AGE (regression), GLEASON (multi-class)
datasets[0] = "smalldata/logreg/prostate.csv";
//CAPSULE (binomial), AGE (regression), GLEASON (multi-class)
responses[0] = new int[] { 1, 2, 8 };
//Iris-type (multi-class)
datasets[1] = "smalldata/iris/iris.csv";
//Iris-type (multi-class)
responses[1] = new int[] { 4 };
HashSet<Long> checkSums = new LinkedHashSet<>();
int testcount = 0;
int count = 0;
for (int i = 0; i < datasets.length; ++i) {
final String dataset = datasets[i];
for (final int resp : responses[i]) {
Frame frame = null, vframe = null;
try {
NFSFileVec nfs = TestUtil.makeNfsFileVec(dataset);
frame = ParseDataset.parse(Key.make(), nfs._key);
NFSFileVec vnfs = TestUtil.makeNfsFileVec(dataset);
vframe = ParseDataset.parse(Key.make(), vnfs._key);
boolean classification = !(i == 0 && resp == 2);
String respname = frame.name(resp);
if (classification && !frame.vec(resp).isCategorical()) {
Vec r = frame.vec(resp).toCategoricalVec();
frame.remove(resp).remove();
frame.add(respname, r);
DKV.put(frame);
Vec vr = vframe.vec(respname).toCategoricalVec();
vframe.remove(respname).remove();
vframe.add(respname, vr);
DKV.put(vframe);
}
if (classification) {
assert (frame.vec(respname).isCategorical());
assert (vframe.vec(respname).isCategorical());
}
for (DeepLearningParameters.Loss loss : new DeepLearningParameters.Loss[] { DeepLearningParameters.Loss.Automatic, DeepLearningParameters.Loss.CrossEntropy, DeepLearningParameters.Loss.Huber, // DeepLearningParameters.Loss.ModifiedHuber,
DeepLearningParameters.Loss.Absolute, DeepLearningParameters.Loss.Quadratic }) {
if (!classification && (loss == DeepLearningParameters.Loss.CrossEntropy || loss == DeepLearningParameters.Loss.ModifiedHuber))
continue;
for (DistributionFamily dist : new DistributionFamily[] { DistributionFamily.AUTO, DistributionFamily.laplace, DistributionFamily.huber, // DistributionFamily.modified_huber,
DistributionFamily.bernoulli, DistributionFamily.gaussian, DistributionFamily.poisson, DistributionFamily.tweedie, DistributionFamily.gamma }) {
if (classification && dist != DistributionFamily.multinomial && dist != DistributionFamily.bernoulli && dist != DistributionFamily.modified_huber)
continue;
if (!classification) {
if (dist == DistributionFamily.multinomial || dist == DistributionFamily.bernoulli || dist == DistributionFamily.modified_huber)
continue;
}
boolean cont = false;
switch(dist) {
case tweedie:
case gamma:
case poisson:
if (loss != DeepLearningParameters.Loss.Automatic)
cont = true;
break;
case huber:
if (loss != DeepLearningParameters.Loss.Huber && loss != DeepLearningParameters.Loss.Automatic)
cont = true;
break;
case laplace:
if (loss != DeepLearningParameters.Loss.Absolute && loss != DeepLearningParameters.Loss.Automatic)
cont = true;
break;
case modified_huber:
if (loss != DeepLearningParameters.Loss.ModifiedHuber && loss != DeepLearningParameters.Loss.Automatic)
cont = true;
break;
case bernoulli:
if (loss != DeepLearningParameters.Loss.CrossEntropy && loss != DeepLearningParameters.Loss.Automatic)
cont = true;
break;
}
if (cont)
continue;
for (boolean elastic_averaging : new boolean[] { true, false }) {
for (boolean replicate : new boolean[] { true, false }) {
for (DeepLearningParameters.Activation activation : new DeepLearningParameters.Activation[] { DeepLearningParameters.Activation.Tanh, DeepLearningParameters.Activation.TanhWithDropout, DeepLearningParameters.Activation.Rectifier, DeepLearningParameters.Activation.RectifierWithDropout, DeepLearningParameters.Activation.Maxout, DeepLearningParameters.Activation.MaxoutWithDropout }) {
boolean reproducible = false;
switch(dist) {
case tweedie:
case gamma:
case poisson:
//don't remember why - probably to force stability
reproducible = true;
default:
}
for (boolean load_balance : new boolean[] { true, false }) {
for (boolean shuffle : new boolean[] { true, false }) {
for (boolean balance_classes : new boolean[] { true, false }) {
for (ClassSamplingMethod csm : new ClassSamplingMethod[] { ClassSamplingMethod.Stratified, ClassSamplingMethod.Uniform }) {
for (int scoretraining : new int[] { 200, 20, 0 }) {
for (int scorevalidation : new int[] { 200, 20, 0 }) {
for (int vf : new int[] { //no validation
0, //same as source
1, //different validation frame
-1 }) {
for (int n_folds : new int[] { 0, 2 }) {
//FIXME: Add back
if (n_folds > 0 && balance_classes)
continue;
for (boolean overwrite_with_best_model : new boolean[] { false, true }) {
for (int train_samples_per_iteration : new int[] { //auto-tune
-2, //N epochs per iteration
-1, //1 epoch per iteration
0, // <1 epoch per iteration
rng.nextInt(200), //>1 epoch per iteration
500 }) {
DeepLearningModel model1 = null, model2 = null;
count++;
if (fraction < rng.nextFloat())
continue;
try {
Log.info("**************************)");
Log.info("Starting test #" + count);
Log.info("**************************)");
final double epochs = 7 + rng.nextDouble() + rng.nextInt(4);
final int[] hidden = new int[] { 3 + rng.nextInt(4), 3 + rng.nextInt(6) };
final double[] hidden_dropout_ratios = activation.name().contains("Hidden") ? new double[] { rng.nextFloat(), rng.nextFloat() } : null;
//no validation
Frame valid = null;
if (//use the same frame for validation
vf == 1)
//use the same frame for validation
valid = frame;
else if (vf == -1)
//different validation frame (here: from the same file)
valid = vframe;
long myseed = rng.nextLong();
boolean replicate2 = rng.nextBoolean();
boolean elastic_averaging2 = rng.nextBoolean();
// build the model, with all kinds of shuffling/rebalancing/sampling
DeepLearningParameters p = new DeepLearningParameters();
{
Log.info("Using seed: " + myseed);
p._train = frame._key;
p._response_column = respname;
p._valid = valid == null ? null : valid._key;
p._hidden = hidden;
p._input_dropout_ratio = 0.1;
p._hidden_dropout_ratios = hidden_dropout_ratios;
p._activation = activation;
// p.best_model_key = best_model_key;
p._overwrite_with_best_model = overwrite_with_best_model;
p._epochs = epochs;
p._loss = loss;
p._distribution = dist;
p._nfolds = n_folds;
p._seed = myseed;
p._train_samples_per_iteration = train_samples_per_iteration;
p._force_load_balance = load_balance;
p._replicate_training_data = replicate;
p._reproducible = reproducible;
p._shuffle_training_data = shuffle;
p._score_training_samples = scoretraining;
p._score_validation_samples = scorevalidation;
p._classification_stop = -1;
p._regression_stop = -1;
p._stopping_rounds = 0;
p._balance_classes = classification && balance_classes;
p._quiet_mode = true;
p._score_validation_sampling = csm;
p._elastic_averaging = elastic_averaging;
// Log.info(new String(p.writeJSON(new AutoBuffer()).buf()).replace(",","\n"));
DeepLearning dl = new DeepLearning(p, Key.<DeepLearningModel>make(Key.make().toString() + "first"));
try {
model1 = dl.trainModel().get();
checkSums.add(model1.checksum());
testcount++;
} catch (Throwable t) {
model1 = DKV.getGet(dl.dest());
if (model1 != null)
Assert.assertTrue(model1._output._job.isCrashed());
throw t;
}
Log.info("Trained for " + model1.epoch_counter + " epochs.");
assert (((p._train_samples_per_iteration <= 0 || p._train_samples_per_iteration >= frame.numRows()) && model1.epoch_counter > epochs) || Math.abs(model1.epoch_counter - epochs) / epochs < 0.20);
// check that iteration is of the expected length - check via when first scoring happens
if (p._train_samples_per_iteration == 0) {
// no sampling - every node does its share of the full data
if (!replicate)
assert ((double) model1._output._scoring_history.get(1, 3) == 1);
else
assert ((double) model1._output._scoring_history.get(1, 3) > 0.7 && (double) model1._output._scoring_history.get(1, 3) < 1.3) : ("First scoring at " + model1._output._scoring_history.get(1, 3) + " epochs, should be closer to 1!" + "\n" + model1.toString());
} else if (p._train_samples_per_iteration == -1) {
// no sampling - every node does its share of the full data
if (!replicate)
assert ((double) model1._output._scoring_history.get(1, 3) == 1);
else // every node passes over the full dataset
{
if (!reproducible)
assert ((double) model1._output._scoring_history.get(1, 3) == H2O.CLOUD.size());
}
}
if (n_folds != 0) {
assert (model1._output._cross_validation_metrics != null);
} else {
assert (model1._output._cross_validation_metrics == null);
}
}
assert (model1.model_info().get_params()._l1 == 0);
assert (model1.model_info().get_params()._l2 == 0);
Assert.assertFalse(model1._output._job.isCrashed());
if (n_folds != 0)
continue;
// Do some more training via checkpoint restart
// For n_folds, continue without n_folds (not yet implemented) - from now on, model2 will have n_folds=0...
DeepLearningParameters p2 = new DeepLearningParameters();
Assert.assertTrue(model1.model_info().get_processed_total() >= frame.numRows() * epochs);
{
p2._checkpoint = model1._key;
p2._distribution = dist;
p2._loss = loss;
p2._nfolds = n_folds;
p2._train = frame._key;
p2._activation = activation;
p2._hidden = hidden;
p2._valid = valid == null ? null : valid._key;
p2._l1 = 1e-3;
p2._l2 = 1e-3;
p2._reproducible = reproducible;
p2._response_column = respname;
p2._overwrite_with_best_model = overwrite_with_best_model;
p2._quiet_mode = true;
//final amount of training epochs
p2._epochs = 2 * epochs;
p2._replicate_training_data = replicate2;
p2._stopping_rounds = 0;
p2._seed = myseed;
// p2._loss = loss; //fall back to default
// p2._distribution = dist; //fall back to default
p2._train_samples_per_iteration = train_samples_per_iteration;
p2._balance_classes = classification && balance_classes;
p2._elastic_averaging = elastic_averaging2;
DeepLearning dl = new DeepLearning(p2);
try {
model2 = dl.trainModel().get();
} catch (Throwable t) {
model2 = DKV.getGet(dl.dest());
if (model2 != null)
Assert.assertTrue(model2._output._job.isCrashed());
throw t;
}
}
Assert.assertTrue(model1._output._job.isDone());
Assert.assertTrue(model2._output._job.isDone());
assert (model1._parms != p2);
assert (model1.model_info().get_params() != model2.model_info().get_params());
assert (model1.model_info().get_params()._l1 == 0);
assert (model1.model_info().get_params()._l2 == 0);
if (!overwrite_with_best_model)
Assert.assertTrue(model2.model_info().get_processed_total() >= frame.numRows() * 2 * epochs);
assert (p != p2);
assert (p != model1.model_info().get_params());
assert (p2 != model2.model_info().get_params());
if (p._loss == DeepLearningParameters.Loss.Automatic) {
assert (p2._loss == DeepLearningParameters.Loss.Automatic);
// assert(model1.model_info().get_params()._loss != DeepLearningParameters.Loss.Automatic);
// assert(model2.model_info().get_params()._loss != DeepLearningParameters.Loss.Automatic);
}
assert (p._hidden_dropout_ratios == null);
assert (p2._hidden_dropout_ratios == null);
if (p._activation.toString().contains("WithDropout")) {
assert (model1.model_info().get_params()._hidden_dropout_ratios != null);
assert (model2.model_info().get_params()._hidden_dropout_ratios != null);
assert (Arrays.equals(model1.model_info().get_params()._hidden_dropout_ratios, model2.model_info().get_params()._hidden_dropout_ratios));
}
assert (p._l1 == 0);
assert (p._l2 == 0);
assert (p2._l1 == 1e-3);
assert (p2._l2 == 1e-3);
assert (model1.model_info().get_params()._l1 == 0);
assert (model1.model_info().get_params()._l2 == 0);
assert (model2.model_info().get_params()._l1 == 1e-3);
assert (model2.model_info().get_params()._l2 == 1e-3);
if (valid == null)
valid = frame;
double threshold;
if (model2._output.isClassifier()) {
Frame pred = null;
Vec labels, predlabels, pred2labels;
try {
pred = model2.score(valid);
DKV.put(Key.make("pred"), pred);
// Build a POJO, validate same results
if (!model2.testJavaScoring(valid, pred, 1e-6)) {
model2.testJavaScoring(valid, pred, 1e-6);
}
Assert.assertTrue(model2.testJavaScoring(valid, pred, 1e-6));
hex.ModelMetrics mm = hex.ModelMetrics.getFromDKV(model2, valid);
double error;
// binary
if (model2._output.nclasses() == 2) {
assert (resp == 1);
threshold = mm.auc_obj().defaultThreshold();
error = mm.auc_obj().defaultErr();
// check that auc.cm() is the right CM
Assert.assertEquals(new ConfusionMatrix(mm.auc_obj().defaultCM(), valid.vec(respname).domain()).err(), error, 1e-15);
// check that calcError() is consistent as well (for CM=null, AUC!=null)
Assert.assertEquals(mm.cm().err(), error, 1e-15);
// check that the labels made with the default threshold are consistent with the CM that's reported by the AUC object
labels = valid.vec(respname);
predlabels = pred.vecs()[0];
ConfusionMatrix cm = buildCM(labels, predlabels);
Log.info("CM from pre-made labels:");
Log.info(cm.toASCII());
if (Math.abs(cm.err() - error) > 2e-2) {
ConfusionMatrix cm2 = buildCM(labels, predlabels);
Log.info(cm2.toASCII());
}
Assert.assertEquals(cm.err(), error, 2e-2);
// confirm that orig CM was made with the right threshold
// manually make labels with AUC-given default threshold
String ast = "(as.factor (> (cols pred [2]) " + threshold + "))";
Frame tmp = Rapids.exec(ast).getFrame();
pred2labels = tmp.vecs()[0];
cm = buildCM(labels, pred2labels);
Log.info("CM from self-made labels:");
Log.info(cm.toASCII());
//AUC-given F1-optimal threshold might not reproduce AUC-given CM-error identically, but should match up to 2%
Assert.assertEquals(cm.err(), error, 2e-2);
tmp.delete();
}
DKV.remove(Key.make("pred"));
} finally {
if (pred != null)
pred.delete();
}
} else //classifier
{
Frame pred = null;
try {
pred = model2.score(valid);
// Build a POJO, validate same results
Assert.assertTrue(model2.testJavaScoring(frame, pred, 1e-6));
} finally {
if (pred != null)
pred.delete();
}
}
Log.info("Parameters combination " + count + ": PASS");
} catch (H2OModelBuilderIllegalArgumentException | IllegalArgumentException ex) {
System.err.println(ex);
throw H2O.fail("should not get here");
} catch (RuntimeException t) {
String msg = // this way we evade null messages
"" + t.getMessage() + (t.getCause() == null ? "" : t.getCause().getMessage());
Assert.assertTrue("Unexpected exception " + t + ": " + msg, msg.contains("unstable"));
} catch (AssertionError ae) {
// test assertions should be preserved
throw ae;
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
if (model1 != null) {
model1.deleteCrossValidationModels();
model1.delete();
}
if (model2 != null) {
model2.deleteCrossValidationModels();
model2.delete();
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
} finally {
if (frame != null)
frame.delete();
if (vframe != null)
vframe.delete();
}
}
}
Log.info("\n\n=============================================");
Log.info("Tested " + testcount + " out of " + count + " parameter combinations.");
Log.info("=============================================");
if (checkSums.size() != testcount) {
Log.info("Only found " + checkSums.size() + " unique checksums.");
}
Assert.assertTrue(checkSums.size() == testcount);
}
Also used :
LinkedHashSet(java.util.LinkedHashSet)
Frame(water.fvec.Frame)
ConfusionMatrix(hex.ConfusionMatrix)
NFSFileVec(water.fvec.NFSFileVec)
DeepLearningParameters(hex.deeplearning.DeepLearningModel.DeepLearningParameters)
ClassSamplingMethod(hex.deeplearning.DeepLearningModel.DeepLearningParameters.ClassSamplingMethod)
Random(java.util.Random)
H2OModelBuilderIllegalArgumentException(water.exceptions.H2OModelBuilderIllegalArgumentException)
DistributionFamily(hex.genmodel.utils.DistributionFamily)
H2OModelBuilderIllegalArgumentException(water.exceptions.H2OModelBuilderIllegalArgumentException)
NFSFileVec(water.fvec.NFSFileVec)
Vec(water.fvec.Vec)
Example 4 with ConfusionMatrix
use of hex.ConfusionMatrix in project h2o-2 by h2oai.
the class AUCData method toHTML.
public boolean toHTML(StringBuilder sb) {
try {
if (actual_domain == null)
actual_domain = new String[] { "false", "true" };
// make local copies to avoid getting clear()'ed out in the middle of printing (can happen for DeepLearning, for example)
String[] my_actual_domain = actual_domain.clone();
String[] my_threshold_criteria = threshold_criteria.clone();
float[] my_threshold_for_criteria = threshold_for_criteria.clone();
float[] my_thresholds = thresholds.clone();
hex.ConfusionMatrix[] my_cms = _cms.clone();
if (my_thresholds == null)
return false;
if (my_threshold_criteria == null)
return false;
if (my_cms == null)
return false;
if (idxCriter == null)
return false;
sb.append("<div>");
DocGen.HTML.section(sb, "<a href=\"http://en.wikipedia.org/wiki/Receiver_operating_characteristic\">Scoring for Binary Classification</a>");
// data for JS
//</script>");
sb.append("\n<script type=\"text/javascript\">");
sb.append("var cms = [\n");
for (hex.ConfusionMatrix cm : _cms) {
StringBuilder tmp = new StringBuilder();
cm.toHTML(tmp, my_actual_domain);
sb.append("\t'" + StringEscapeUtils.escapeJavaScript(tmp.toString()) + "',\n");
}
sb.append("];\n");
//which one
sb.append("var criterion = " + threshold_criterion.ordinal() + ";\n");
sb.append("var criteria = [");
for (String c : my_threshold_criteria) sb.append("\"" + c + "\",");
sb.append(" ];\n");
sb.append("var thresholds = [");
for (double t : my_threshold_for_criteria) sb.append((float) t + ",");
sb.append(" ];\n");
sb.append("var F1_values = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].F1() + ",");
sb.append(" ];\n");
sb.append("var F2_values = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].F2() + ",");
sb.append(" ];\n");
sb.append("var F0point5_values = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].F0point5() + ",");
sb.append(" ];\n");
sb.append("var accuracy = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].accuracy() + ",");
sb.append(" ];\n");
sb.append("var error = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].err() + ",");
sb.append(" ];\n");
sb.append("var precision = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].precision() + ",");
sb.append(" ];\n");
sb.append("var recall = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].recall() + ",");
sb.append(" ];\n");
sb.append("var specificity = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].specificity() + ",");
sb.append(" ];\n");
sb.append("var mcc = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].mcc() + ",");
sb.append(" ];\n");
sb.append("var max_per_class_error = [");
for (int i = 0; i < my_cms.length; ++i) sb.append((float) my_cms[i].max_per_class_error() + ",");
sb.append(" ];\n");
sb.append("var idxCriter = [");
for (int i : idxCriter) sb.append(i + ",");
sb.append(" ];\n");
sb.append("</script>\n");
// Selection of threshold criterion
sb.append("\n<div><b>Threshold criterion:</b></div><select id='threshold_select' onchange='set_criterion(this.value, idxCriter[this.value])'>\n");
for (int i = 0; i < my_threshold_criteria.length; ++i) sb.append("\t<option value='" + i + "'" + (i == threshold_criterion.ordinal() ? "selected='selected'" : "") + ">" + my_threshold_criteria[i] + "</option>\n");
sb.append("</select>\n");
sb.append("</div>");
DocGen.HTML.arrayHead(sb);
sb.append("<th>AUC</th>");
sb.append("<th>Gini</th>");
sb.append("<th id='threshold_criterion'>Threshold for " + threshold_criterion.toString().replace("_", " ") + "</th>");
sb.append("<th>F1 </th>");
sb.append("<th>Accuracy </th>");
sb.append("<th>Error </th>");
sb.append("<th>Precision </th>");
sb.append("<th>Recall </th>");
sb.append("<th>Specificity</th>");
sb.append("<th>MCC</th>");
sb.append("<th>Max per class Error</th>");
sb.append("<tr class='warning'>");
sb.append("<td>" + String.format("%.5f", AUC()) + "</td>" + "<td>" + String.format("%.5f", Gini()) + "</td>" + "<td id='threshold'>" + String.format("%g", threshold()) + "</td>" + "<td id='F1_value'>" + String.format("%.7f", F1()) + "</td>" + "<td id='accuracy'>" + String.format("%.7f", accuracy()) + "</td>" + "<td id='error'>" + String.format("%.7f", err()) + "</td>" + "<td id='precision'>" + String.format("%.7f", precision()) + "</td>" + "<td id='recall'>" + String.format("%.7f", recall()) + "</td>" + "<td id='specificity'>" + String.format("%.7f", specificity()) + "</td>" + "<td id='mcc'>" + String.format("%.7f", mcc()) + "</td>" + "<td id='max_per_class_error'>" + String.format("%.7f", max_per_class_error()) + "</td>");
DocGen.HTML.arrayTail(sb);
// sb.append("<div id='BestConfusionMatrix'>");
// CM().toHTML(sb, actual_domain);
// sb.append("</div>");
sb.append("<table><tr><td>");
plotROC(sb);
sb.append("</td><td id='ConfusionMatrix'>");
CM().toHTML(sb, my_actual_domain);
sb.append("</td></tr>");
sb.append("<tr><td><h5>Threshold:</h5></div><select id=\"select\" onchange='show_cm(this.value)'>\n");
for (int i = 0; i < my_cms.length; ++i) sb.append("\t<option value='" + i + "'" + (my_thresholds[i] == threshold() ? "selected='selected'" : "") + ">" + my_thresholds[i] + "</option>\n");
sb.append("</select></td></tr>");
sb.append("</td>");
sb.append("</table>");
sb.append("\n<script type=\"text/javascript\">");
sb.append("function show_cm(i){\n");
sb.append("\t" + "document.getElementById('ConfusionMatrix').innerHTML = cms[i];\n");
sb.append("\t" + "document.getElementById('F1_value').innerHTML = F1_values[i];\n");
sb.append("\t" + "document.getElementById('accuracy').innerHTML = accuracy[i];\n");
sb.append("\t" + "document.getElementById('error').innerHTML = error[i];\n");
sb.append("\t" + "document.getElementById('precision').innerHTML = precision[i];\n");
sb.append("\t" + "document.getElementById('recall').innerHTML = recall[i];\n");
sb.append("\t" + "document.getElementById('specificity').innerHTML = specificity[i];\n");
sb.append("\t" + "document.getElementById('mcc').innerHTML = mcc[i];\n");
sb.append("\t" + "document.getElementById('max_per_class_error').innerHTML = max_per_class_error[i];\n");
sb.append("\t" + "update(dataset);\n");
sb.append("}\n");
sb.append("function set_criterion(i, idx){\n");
sb.append("\t" + "criterion = i;\n");
// sb.append("\t" + "document.getElementById('BestConfusionMatrix').innerHTML = cms[idx];\n");
sb.append("\t" + "document.getElementById('threshold_criterion').innerHTML = \" Threshold for \" + criteria[i];\n");
sb.append("\t" + "document.getElementById('threshold').innerHTML = thresholds[i];\n");
sb.append("\t" + "show_cm(idx);\n");
sb.append("\t" + "document.getElementById(\"select\").selectedIndex = idx;\n");
sb.append("\t" + "update(dataset);\n");
sb.append("}\n");
sb.append("</script>\n");
return true;
} catch (Exception ex) {
return false;
}
}
Also used :
ConfusionMatrix(hex.ConfusionMatrix)
ConfusionMatrix(hex.ConfusionMatrix)
Example 5 with ConfusionMatrix
use of hex.ConfusionMatrix in project h2o-2 by h2oai.
the class SharedTreeModelBuilder method doScoring.
protected TM doScoring(TM model, Frame fTrain, DTree[] ktrees, int tid, DTree.TreeModel.TreeStats tstats, boolean finalScoring, boolean oob, boolean build_tree_one_node) {
long now = System.currentTimeMillis();
if (_firstScore == 0)
_firstScore = now;
long sinceLastScore = now - _timeLastScoreStart;
Score sc = null;
// If validation is specified we use a model for scoring, so we need to update it!
// First we save model with trees (i.e., make them available for scoring)
// and then update it with resulting error
model = makeModel(model, ktrees, tstats);
model.update(self());
// Now model already contains tid-trees in serialized form
if (score_each_iteration || finalScoring || // Score every time for 4 secs
(now - _firstScore < 4000) || // Throttle scoring to keep the cost sane; limit to a 10% duty cycle & every 4 secs
(// Limit scoring updates to every 4sec
sinceLastScore > 4000 && (double) (_timeLastScoreEnd - _timeLastScoreStart) / sinceLastScore < 0.1)) {
// 10% duty cycle
_timeLastScoreStart = now;
// Perform scoring - first get adapted validation response
Response2CMAdaptor vadaptor = getValidAdaptor();
sc = new Score().doIt(model, fTrain, vadaptor, oob, build_tree_one_node).report(logTag(), tid, ktrees);
_timeLastScoreEnd = System.currentTimeMillis();
}
// Compute variable importance for this tree if necessary
VarImp varimp = null;
if (importance && ktrees != null) {
// compute this tree votes but skip the first scoring call which is done over empty forest
Timer vi_timer = new Timer();
varimp = doVarImpCalc(model, ktrees, tid - 1, fTrain, false);
Log.info(logTag(), "Computation of variable importance with " + tid + "th-tree took: " + vi_timer.toString());
}
// Double update - after scoring
model = makeModel(model, sc == null ? Double.NaN : sc.mse(), sc == null ? null : (_nclass > 1 ? new ConfusionMatrix(sc._cm) : null), varimp, sc == null ? null : (_nclass == 2 ? makeAUC(toCMArray(sc._cms), ModelUtils.DEFAULT_THRESHOLDS) : null));
model.update(self());
return model;
}
Also used :
ConfusionMatrix(hex.ConfusionMatrix)
VarImp(hex.VarImp)
Aggregations
ConfusionMatrix (hex.ConfusionMatrix)6
Frame (water.fvec.Frame)2
Vec (water.fvec.Vec)2
VarImp (hex.VarImp)1
DeepLearningParameters (hex.deeplearning.DeepLearningModel.DeepLearningParameters)1
ClassSamplingMethod (hex.deeplearning.DeepLearningModel.DeepLearningParameters.ClassSamplingMethod)1
DistributionFamily (hex.genmodel.utils.DistributionFamily)1
LinkedHashSet (java.util.LinkedHashSet)1
Random (java.util.Random)1
water (water)1
water.api (water.api)1
AUC (water.api.AUC)1
H2OModelBuilderIllegalArgumentException (water.exceptions.H2OModelBuilderIllegalArgumentException)1
Chunk (water.fvec.Chunk)1
NFSFileVec (water.fvec.NFSFileVec)1
NewChunk (water.fvec.NewChunk)1
