Examples with ModelMetricsMultinomialGLM hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM used on opensource projects

Example 1 with ModelMetricsMultinomialGLM

use of hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM in project h2o-3 by h2oai.

the class GLMMetricBuilder method makeModelMetrics.

@Override
public ModelMetrics makeModelMetrics(Model m, Frame f, Frame adaptedFrame, Frame preds) {
    GLMModel gm = (GLMModel) m;
    computeAIC();
    ModelMetrics metrics = _metricBuilder.makeModelMetrics(gm, f, null, null);
    if (_glmf._family == Family.binomial) {
        ModelMetricsBinomial metricsBinommial = (ModelMetricsBinomial) metrics;
        GainsLift gl = null;
        if (preds != null) {
            Vec resp = f.vec(m._parms._response_column);
            Vec weights = f.vec(m._parms._weights_column);
            if (resp != null) {
                gl = new GainsLift(preds.lastVec(), resp, weights);
                gl.exec(m._output._job);
            }
        }
        metrics = new ModelMetricsBinomialGLM(m, f, metrics._nobs, metrics._MSE, _domain, metricsBinommial._sigma, metricsBinommial._auc, metricsBinommial._logloss, residualDeviance(), null_devince, _aic, nullDOF(), resDOF(), gl);
    } else if (_glmf._family == Family.multinomial) {
        ModelMetricsMultinomial metricsMultinomial = (ModelMetricsMultinomial) metrics;
        metrics = new ModelMetricsMultinomialGLM(m, f, metricsMultinomial._nobs, metricsMultinomial._MSE, metricsMultinomial._domain, metricsMultinomial._sigma, metricsMultinomial._cm, metricsMultinomial._hit_ratios, metricsMultinomial._logloss, residualDeviance(), null_devince, _aic, nullDOF(), resDOF());
    } else {
        ModelMetricsRegression metricsRegression = (ModelMetricsRegression) metrics;
        metrics = new ModelMetricsRegressionGLM(m, f, metricsRegression._nobs, metricsRegression._MSE, metricsRegression._sigma, metricsRegression._mean_absolute_error, metricsRegression._root_mean_squared_log_error, residualDeviance(), residualDeviance() / _wcount, null_devince, _aic, nullDOF(), resDOF());
    }
    // Update the metrics in-place with the GLM version
    return gm.addModelMetrics(metrics);
}

Example 2 with ModelMetricsMultinomialGLM

use of hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM in project h2o-3 by h2oai.

the class GLMBasicTestMultinomial method testCovtypeBasic.

@Test
public void testCovtypeBasic() {
    GLMParameters params = new GLMParameters(Family.multinomial);
    GLMModel model = null;
    Frame preds = null;
    Vec weights = _covtype.anyVec().makeCon(1);
    Key k = Key.<Frame>make("cov_with_weights");
    Frame f = new Frame(k, _covtype.names(), _covtype.vecs());
    f.add("weights", weights);
    DKV.put(f);
    try {
        params._response_column = "C55";
        params._train = k;
        params._valid = _covtype._key;
        params._lambda = new double[] { 4.881e-05 };
        params._alpha = new double[] { 1 };
        params._objective_epsilon = 1e-6;
        params._beta_epsilon = 1e-4;
        params._weights_column = "weights";
        params._missing_values_handling = DeepLearningModel.DeepLearningParameters.MissingValuesHandling.Skip;
        double[] alpha = new double[] { 1 };
        double[] expected_deviance = new double[] { 25499.76 };
        double[] lambda = new double[] { 2.544750e-05 };
        for (Solver s : new Solver[] { Solver.IRLSM, Solver.COORDINATE_DESCENT, Solver.L_BFGS }) {
            System.out.println("solver = " + s);
            params._solver = s;
            params._max_iterations = params._solver == Solver.L_BFGS ? 300 : 10;
            for (int i = 0; i < alpha.length; ++i) {
                params._alpha[0] = alpha[i];
                params._lambda[0] = lambda[i];
                model = new GLM(params).trainModel().get();
                System.out.println(model._output._model_summary);
                System.out.println(model._output._training_metrics);
                System.out.println(model._output._validation_metrics);
                assertTrue(model._output._training_metrics.equals(model._output._validation_metrics));
                assertTrue(((ModelMetricsMultinomialGLM) model._output._training_metrics)._resDev <= expected_deviance[i] * 1.1);
                preds = model.score(_covtype);
                ModelMetricsMultinomialGLM mmTrain = (ModelMetricsMultinomialGLM) hex.ModelMetricsMultinomial.getFromDKV(model, _covtype);
                assertTrue(model._output._training_metrics.equals(mmTrain));
                model.delete();
                model = null;
                preds.delete();
                preds = null;
            }
        }
    } finally {
        weights.remove();
        DKV.remove(k);
        if (model != null)
            model.delete();
        if (preds != null)
            preds.delete();
    }
}

Example 3 with ModelMetricsMultinomialGLM

use of hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM in project h2o-3 by h2oai.

the class GLMBasicTestMultinomial method testCovtypeNoIntercept.

@Test
public void testCovtypeNoIntercept() {
    GLMParameters params = new GLMParameters(Family.multinomial);
    GLMModel model = null;
    Frame preds = null;
    Vec weights = _covtype.anyVec().makeCon(1);
    Key k = Key.<Frame>make("cov_with_weights");
    Frame f = new Frame(k, _covtype.names(), _covtype.vecs());
    f.add("weights", weights);
    DKV.put(f);
    try {
        params._response_column = "C55";
        params._train = k;
        params._valid = _covtype._key;
        params._objective_epsilon = 1e-6;
        params._beta_epsilon = 1e-4;
        params._weights_column = "weights";
        params._missing_values_handling = DeepLearningModel.DeepLearningParameters.MissingValuesHandling.Skip;
        params._intercept = false;
        double[] alpha = new double[] { 0, .5, .1 };
        Solver s = Solver.L_BFGS;
        System.out.println("solver = " + s);
        params._solver = s;
        params._max_iterations = 5000;
        for (int i = 0; i < alpha.length; ++i) {
            params._alpha = new double[] { alpha[i] };
            //        params._lambda[0] = lambda[i];
            model = new GLM(params).trainModel().get();
            System.out.println(model.coefficients());
            //        Assert.assertEquals(0,model.coefficients().get("Intercept"),0);
            double[][] bs = model._output.getNormBetaMultinomial();
            for (double[] b : bs) Assert.assertEquals(0, b[b.length - 1], 0);
            System.out.println(model._output._model_summary);
            System.out.println(model._output._training_metrics);
            System.out.println(model._output._validation_metrics);
            preds = model.score(_covtype);
            ModelMetricsMultinomialGLM mmTrain = (ModelMetricsMultinomialGLM) hex.ModelMetricsMultinomial.getFromDKV(model, _covtype);
            assertTrue(model._output._training_metrics.equals(mmTrain));
            model.delete();
            model = null;
            preds.delete();
            preds = null;
        }
    } finally {
        weights.remove();
        DKV.remove(k);
        if (model != null)
            model.delete();
        if (preds != null)
            preds.delete();
    }
}

Example 4 with ModelMetricsMultinomialGLM

use of hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM in project h2o-3 by h2oai.

the class GLMBasicTestMultinomial method testCovtypeMinActivePredictors.

@Test
public void testCovtypeMinActivePredictors() {
    GLMParameters params = new GLMParameters(Family.multinomial);
    GLMModel model = null;
    Frame preds = null;
    try {
        params._response_column = "C55";
        params._train = _covtype._key;
        params._valid = _covtype._key;
        params._lambda = new double[] { 4.881e-05 };
        params._alpha = new double[] { 1 };
        params._objective_epsilon = 1e-6;
        params._beta_epsilon = 1e-4;
        params._max_active_predictors = 50;
        params._max_iterations = 10;
        double[] alpha = new double[] { .99 };
        double expected_deviance = 33000;
        double[] lambda = new double[] { 2.544750e-05 };
        Solver s = Solver.COORDINATE_DESCENT;
        System.out.println("solver = " + s);
        params._solver = s;
        model = new GLM(params).trainModel().get();
        System.out.println(model._output._model_summary);
        System.out.println(model._output._training_metrics);
        System.out.println(model._output._validation_metrics);
        System.out.println("rank = " + model._output.rank() + ", max active preds = " + params._max_active_predictors);
        assertTrue(model._output.rank() < params._max_active_predictors + model._output.nclasses());
        assertTrue(model._output._training_metrics.equals(model._output._validation_metrics));
        assertTrue(((ModelMetricsMultinomialGLM) model._output._training_metrics)._resDev <= expected_deviance * 1.1);
        preds = model.score(_covtype);
        ModelMetricsMultinomialGLM mmTrain = (ModelMetricsMultinomialGLM) hex.ModelMetricsMultinomial.getFromDKV(model, _covtype);
        assertTrue(model._output._training_metrics.equals(mmTrain));
        model.delete();
        model = null;
        preds.delete();
        preds = null;
    } finally {
        if (model != null)
            model.delete();
        if (preds != null)
            preds.delete();
    }
}

Example 5 with ModelMetricsMultinomialGLM

use of hex.ModelMetricsBinomialGLM.ModelMetricsMultinomialGLM in project h2o-3 by h2oai.

the class GLMBasicTestMultinomial method testCovtypeLS.

@Test
public void testCovtypeLS() {
    GLMParameters params = new GLMParameters(Family.multinomial);
    GLMModel model = null;
    Frame preds = null;
    try {
        double expected_deviance = 33000;
        params._nlambdas = 3;
        params._response_column = "C55";
        params._train = _covtype._key;
        params._valid = _covtype._key;
        params._alpha = new double[] { .99 };
        params._objective_epsilon = 1e-6;
        params._beta_epsilon = 1e-4;
        params._max_active_predictors = 50;
        params._max_iterations = 500;
        params._solver = Solver.AUTO;
        params._lambda_search = true;
        model = new GLM(params).trainModel().get();
        System.out.println(model._output._training_metrics);
        System.out.println(model._output._validation_metrics);
        assertTrue(model._output._training_metrics.equals(model._output._validation_metrics));
        preds = model.score(_covtype);
        ModelMetricsMultinomialGLM mmTrain = (ModelMetricsMultinomialGLM) hex.ModelMetricsMultinomial.getFromDKV(model, _covtype);
        assertTrue(model._output._training_metrics.equals(mmTrain));
        assertTrue(((ModelMetricsMultinomialGLM) model._output._training_metrics)._resDev <= expected_deviance);
        System.out.println(model._output._model_summary);
        model.delete();
        model = null;
        preds.delete();
        preds = null;
    } finally {
        if (model != null)
            model.delete();
        if (preds != null)
            preds.delete();
    }
}