Examples with YMUTask hex.glm.GLMTask.YMUTask used on opensource projects

Example 1 with YMUTask

use of hex.glm.GLMTask.YMUTask in project h2o-2 by h2oai.

the class GLM2 method run.

public void run(boolean doLog, H2OCountedCompleter cmp) {
    if (doLog)
        logStart();
    // just fork off the nfolds+1 tasks and wait for the results
    assert alpha.length == 1;
    start_time = System.currentTimeMillis();
    if (nlambdas == -1)
        nlambdas = 100;
    if (lambda_search && nlambdas <= 1)
        throw new IllegalArgumentException(LogInfo("GLM2: nlambdas must be > 1 when running with lambda search."));
    Futures fs = new Futures();
    Key dst = dest();
    new YMUTask(GLM2.this.self(), _srcDinfo, n_folds, new H2OCallback<YMUTask>(cmp) {

        @Override
        public String toString() {
            return "YMUTask callback. completer = " + getCompleter() != null ? "null" : getCompleter().toString();
        }

        @Override
        public void callback(final YMUTask ymut) {
            if (ymut._ymin == ymut._ymax)
                throw new IllegalArgumentException(LogInfo("GLM2: attempted to run with constant response. Response == " + ymut._ymin + " for all rows in the training set."));
            if (ymut.nobs() == 0)
                throw new IllegalArgumentException(LogInfo("GLM2: got no active rows in the dataset after discarding rows with NAs"));
            _ymu = ymut.ymu();
            _nobs = ymut.nobs();
            if (_glm.family == Family.binomial && prior != -1 && prior != _ymu && !Double.isNaN(prior)) {
                _iceptAdjust = -Math.log(_ymu * (1 - prior) / (prior * (1 - _ymu)));
            } else
                prior = _ymu;
            H2OCountedCompleter cmp = (H2OCountedCompleter) getCompleter();
            cmp.addToPendingCount(1);
            // public GLMIterationTask(int noff, Key jobKey, DataInfo dinfo, GLMParams glm, boolean computeGram, boolean validate, boolean computeGradient, double [] beta, double ymu, double reg, float [] thresholds, H2OCountedCompleter cmp) {
            new GLMIterationTask(_noffsets, GLM2.this.self(), _srcDinfo, _glm, false, true, true, nullModelBeta(_srcDinfo, _ymu), _ymu, 1.0 / _nobs, thresholds, new H2OCallback<GLMIterationTask>(cmp) {

                @Override
                public String toString() {
                    return "LMAXTask callback. completer = " + (getCompleter() != null ? "NULL" : getCompleter().toString());
                }

                @Override
                public void callback(final GLMIterationTask glmt) {
                    double[] beta = glmt._beta;
                    if (beta_start == null) {
                        beta_start = beta;
                    }
                    _nullDeviance = glmt._val.residualDeviance();
                    _currentLambda = lambda_max = Math.max(Utils.maxValue(glmt._grad), -Utils.minValue(glmt._grad)) / Math.max(1e-3, alpha[0]);
                    _lastResult = makeIterationInfo(0, glmt, null, glmt.gradient(0, 0));
                    GLMModel model = new GLMModel(GLM2.this, dest(), _srcDinfo, _glm, glmt._val, beta_epsilon, alpha[0], lambda_max, _ymu, prior);
                    model.start_training(start_time);
                    if (lambda_search) {
                        assert !Double.isNaN(lambda_max) : LogInfo("running lambda_value search, but don't know what is the lambda_value max!");
                        model = addLmaxSubmodel(model, glmt._val, beta);
                        if (nlambdas == -1) {
                            lambda = null;
                        } else {
                            if (lambda_min_ratio == -1)
                                lambda_min_ratio = _nobs > 25 * _srcDinfo.fullN() ? 1e-4 : 1e-2;
                            final double d = Math.pow(lambda_min_ratio, 1.0 / (nlambdas - 1));
                            if (nlambdas == 0)
                                throw new IllegalArgumentException("nlambdas must be > 0 when running lambda search.");
                            lambda = new double[nlambdas];
                            lambda[0] = lambda_max;
                            if (nlambdas == 1)
                                throw new IllegalArgumentException("Number of lambdas must be > 1 when running with lambda_search!");
                            for (int i = 1; i < lambda.length; ++i) lambda[i] = lambda[i - 1] * d;
                            lambda_min = lambda[lambda.length - 1];
                            max_iter = MAX_ITERATIONS_PER_LAMBDA * nlambdas;
                        }
                        _runAllLambdas = false;
                    } else {
                        if (lambda == null || lambda.length == 0)
                            lambda = new double[] { DEFAULT_LAMBDA };
                        int i = 0;
                        while (i < lambda.length && lambda[i] > lambda_max) ++i;
                        if (i == lambda.length)
                            throw new IllegalArgumentException("Given lambda(s) are all > lambda_max = " + lambda_max + ", have nothing to run with. lambda = " + Arrays.toString(lambda));
                        if (i > 0) {
                            model.addWarning("Removed " + i + " lambdas greater than lambda_max.");
                            lambda = Utils.append(new double[] { lambda_max }, Arrays.copyOfRange(lambda, i, lambda.length));
                            addLmaxSubmodel(model, glmt._val, beta);
                        }
                    }
                    model.delete_and_lock(self());
                    lambda_min = lambda[lambda.length - 1];
                    if (n_folds > 1) {
                        final H2OCountedCompleter futures = new H2OEmptyCompleter();
                        final GLM2[] xvals = new GLM2[n_folds + 1];
                        futures.addToPendingCount(xvals.length - 2);
                        for (int i = 0; i < xvals.length; ++i) {
                            xvals[i] = (GLM2) GLM2.this.clone();
                            xvals[i].n_folds = 0;
                            xvals[i].standardize = standardize;
                            xvals[i].family = family;
                            xvals[i].link = link;
                            xvals[i].beta_epsilon = beta_epsilon;
                            xvals[i].max_iter = max_iter;
                            xvals[i].variable_importances = variable_importances;
                            if (i != 0) {
                                xvals[i]._srcDinfo = _srcDinfo.getFold(i - 1, n_folds);
                                xvals[i].destination_key = Key.make(dest().toString() + "_xval_" + i, (byte) 1, Key.HIDDEN_USER_KEY, H2O.SELF);
                                xvals[i]._nobs = ymut.nobs(i - 1);
                                xvals[i]._ymu = ymut.ymu(i - 1);
                                final int fi = i;
                                final double ymu = ymut.ymu(fi - 1);
                                // new GLMIterationTask(offset_cols.length,GLM2.this.self(), _srcDinfo, _glm, false, true, true,nullModelBeta(),_ymu,1.0/_nobs, thresholds, new H2OCallback<GLMIterationTask>(cmp){
                                new GLMIterationTask(_noffsets, self(), xvals[i]._srcDinfo, _glm, false, true, true, nullModelBeta(xvals[fi]._srcDinfo, ymu), ymu, 1.0 / ymut.nobs(fi - 1), thresholds, new H2OCallback<GLMIterationTask>(futures) {

                                    @Override
                                    public String toString() {
                                        return "Xval LMAXTask callback., completer = " + getCompleter() == null ? "null" : getCompleter().toString();
                                    }

                                    @Override
                                    public void callback(GLMIterationTask t) {
                                        xvals[fi].beta_start = t._beta;
                                        xvals[fi]._currentLambda = xvals[fi].lambda_max = Math.max(Utils.maxValue(glmt._grad), -Utils.minValue(glmt._grad)) / Math.max(1e-3, alpha[0]);
                                        assert xvals[fi].lambda_max > 0;
                                        xvals[fi]._lastResult = makeIterationInfo(0, t, null, t.gradient(alpha[0], 0));
                                        //.delete_and_lock(self());
                                        GLMModel m = new GLMModel(GLM2.this, xvals[fi].destination_key, xvals[fi]._srcDinfo, _glm, t._val, beta_epsilon, alpha[0], xvals[fi].lambda_max, xvals[fi]._ymu, prior);
                                        m.submodels = new Submodel[] { new Submodel(xvals[fi].lambda_max, t._beta, t._beta, 0, 0, t._beta.length >= sparseCoefThreshold) };
                                        m.submodels[0].validation = t._val;
                                        assert t._val != null;
                                        m.setSubmodelIdx(0);
                                        m.delete_and_lock(self());
                                        if (xvals[fi].lambda_max > lambda_max) {
                                            futures.addToPendingCount(1);
                                            new ParallelGLMs(GLM2.this, new GLM2[] { xvals[fi] }, lambda_max, 1, futures).fork();
                                        }
                                    }
                                }).asyncExec(xvals[i]._srcDinfo._adaptedFrame);
                            }
                        }
                        _xvals = xvals;
                        futures.join();
                    }
                    getCompleter().addToPendingCount(1);
                    nextLambda(nextLambdaValue(), new LambdaIteration(getCompleter()));
                }
            }).asyncExec(_srcDinfo._adaptedFrame);
        }
    }).asyncExec(_srcDinfo._adaptedFrame);
}