Examples with GradientSolver hex.optimization.OptimizationUtils.GradientSolver used on opensource projects

Example 1 with GradientSolver

use of hex.optimization.OptimizationUtils.GradientSolver in project h2o-3 by h2oai.

the class LineSearchTest method testMoreThuenteMethod.

@Test
public void testMoreThuenteMethod() {
    GradientSolver f = new GradientSolver() {

        @Override
        public GradientInfo getGradient(double[] beta) {
            GradientInfo ginfo = new GradientInfo(0, new double[1]);
            double x = beta[0];
            double b = 2;
            double xx = x * x;
            ginfo._gradient[0] = (xx - b) / ((b + xx) * (b + xx));
            ginfo._objVal = -x / (xx + b);
            return ginfo;
        }

        @Override
        public GradientInfo getObjective(double[] beta) {
            return getGradient(beta);
        }
    };
    double stp = 1;
    double x = 100;
    MoreThuente ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    boolean succ = ls.evaluate(new double[] { -1 });
    assertTrue(succ);
    assertEquals(1, ls._returnStatus);
    assertEquals(18, ls.nfeval());
    assertEquals(-0.35355, ls.ginfo()._objVal, 1e-5);
    assertEquals(98586, Math.round(1000 * ls.step()), 1e-5);
    x = 0;
    stp = 100;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(5, ls.nfeval());
    assertEquals(-0.34992, ls.ginfo()._objVal, 1e-5);
    assertEquals(1.6331, ls.step(), 1e-5);
    x = 0;
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(1, ls.nfeval());
    x = 0;
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertEquals(ls._returnStatus, 1);
    assertTrue(succ);
    assertEquals(4, ls.nfeval());
    assertEquals(37, Math.round(ls.step()));
    x = 0;
    stp = 1e-3;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertEquals(ls._returnStatus, 1);
    assertTrue(succ);
    assertEquals(6, ls.nfeval());
    assertEquals(14, Math.round(10 * ls.step()));
    f = new GradientSolver() {

        @Override
        public GradientInfo getGradient(double[] beta) {
            GradientInfo ginfo = new GradientInfo(0, new double[1]);
            double x = beta[0];
            double b = 0.004;
            ginfo._objVal = Math.pow(x + b, 5) - 2 * Math.pow(x + b, 4);
            ginfo._gradient[0] = Math.pow(b + x, 3) * (5 * (b + x) - 8);
            return ginfo;
        }

        @Override
        public GradientInfo getObjective(double[] beta) {
            return getGradient(beta);
        }
    };
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(12, ls.nfeval());
    assertEquals(16, Math.round(10 * ls.step()));
    stp = 0.1;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(8, ls.nfeval());
    assertEquals(16, Math.round(10 * ls.step()));
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(8, ls.nfeval());
    assertEquals(16, Math.round(10 * ls.step()));
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(11, ls.nfeval());
    assertEquals(16, Math.round(10 * ls.step()));
    f = new GradientSolver() {

        final double beta = 0.01;

        final double l = 39;

        double phi0(double x) {
            if (x <= 1 - beta) {
                return 1 - x;
            } else if (x >= 1 + beta) {
                return x - 1;
            } else {
                return .5 * ((x - 1) * (x - 1) / beta + beta);
            }
        }

        double phi0Prime(double x) {
            if (x <= 1 - beta) {
                return -1;
            } else if (x >= 1 + beta) {
                return 1;
            } else {
                // .5*((x-1)*(x-1)/beta + beta);
                return (x - 1) / beta;
            }
        }

        @Override
        public GradientInfo getGradient(double[] ary) {
            GradientInfo ginfo = new GradientInfo(0, new double[1]);
            double x = ary[0];
            double a = 2 * (1 - beta) / (Math.PI * l);
            double b = .5 * l * Math.PI;
            ginfo._objVal = phi0(x) + a * Math.sin(b * x);
            ginfo._gradient[0] = phi0Prime(x) + a * b * Math.cos(b * x);
            return ginfo;
        }

        @Override
        public GradientInfo getObjective(double[] beta) {
            return getGradient(beta);
        }
    };
    stp = 0.001;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertEquals(ls._returnStatus, 1);
    assertTrue(succ);
    assertEquals(12, ls.nfeval());
    assertEquals(10, Math.round(10 * ls.step()));
    stp = 0.1;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(12, ls.nfeval());
    assertEquals(10, Math.round(10 * ls.step()));
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertEquals(ls._returnStatus, 1);
    assertTrue(succ);
    assertEquals(10, ls.nfeval());
    assertEquals(10, Math.round(10 * ls.step()));
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-1, 1e-1, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(13, ls.nfeval());
    assertEquals(10, Math.round(10 * ls.step()));
    f = new F(1e-3, 1e-3);
    stp = 0.001;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(4, ls.nfeval());
    assertEquals(9, Math.round(100 * ls.step()));
    stp = 0.1;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(1, ls.nfeval());
    assertEquals(10, Math.round(100 * ls.step()));
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(3, ls.nfeval());
    assertEquals(35, Math.round(100 * ls.step()));
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(4, ls.nfeval());
    assertEquals(83, Math.round(100 * ls.step()));
    f = new F(0.01, 1e-3);
    stp = 0.001;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(6, ls.nfeval());
    assertEquals(75, Math.round(1000 * ls.step()));
    stp = 0.1;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(3, ls.nfeval());
    assertEquals(78, Math.round(1000 * ls.step()));
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(7, ls.nfeval());
    assertEquals(73, Math.round(1000 * ls.step()));
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(8, ls.nfeval());
    assertEquals(76, Math.round(1000 * ls.step()));
    f = new F(1e-3, 0.01);
    stp = 0.001;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(13, ls.nfeval());
    assertEquals(93, Math.round(100 * ls.step()));
    stp = 0.1;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(11, ls.nfeval());
    assertEquals(93, Math.round(100 * ls.step()));
    stp = 10;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(8, ls.nfeval());
    assertEquals(92, Math.round(100 * ls.step()));
    stp = 1000;
    ls = new OptimizationUtils.MoreThuente(f, new double[] { x }, f.getGradient(new double[] { x }), 1e-3, 1e-3, 1e-5).setInitialStep(stp);
    succ = ls.evaluate(new double[] { 1 });
    assertTrue(succ);
    assertEquals(ls._returnStatus, 1);
    assertEquals(11, ls.nfeval());
    assertEquals(92, Math.round(100 * ls.step()));
}

Example 2 with GradientSolver

use of hex.optimization.OptimizationUtils.GradientSolver in project h2o-3 by h2oai.

the class ComputationState method gslvrMultinomial.

public GradientSolver gslvrMultinomial(final int c) {
    final double[] fullbeta = _beta.clone();
    return new GradientSolver() {

        @Override
        public GradientInfo getGradient(double[] beta) {
            fillSubRange(_activeData.fullN() + 1, c, _activeDataMultinomial[c].activeCols(), beta, fullbeta);
            GLMGradientInfo fullGinfo = _gslvr.getGradient(fullbeta);
            return new GLMSubsetGinfo(fullGinfo, _activeData.fullN() + 1, c, _activeDataMultinomial[c].activeCols());
        }

        @Override
        public GradientInfo getObjective(double[] beta) {
            return getGradient(beta);
        }
    };
}

Example 3 with GradientSolver

use of hex.optimization.OptimizationUtils.GradientSolver in project h2o-3 by h2oai.

the class L_BFGS_Test method rosenbrock.

// test on Rosenbrock's function (known optimum at (a,a^2), minimum = 0)
@Test
public void rosenbrock() {
    final double a = 1, b = 100;
    GradientSolver gs = new GradientSolver() {

        @Override
        public GradientInfo getGradient(double[] beta) {
            final double[] g = new double[2];
            final double x = beta[0];
            final double y = beta[1];
            final double xx = x * x;
            g[0] = -2 * a + 2 * x - 4 * b * (y * x - x * xx);
            g[1] = 2 * b * (y - xx);
            double objVal = (a - x) * (a - x) + b * (y - xx) * (y - xx);
            return new GradientInfo(objVal, g);
        }

        @Override
        public GradientInfo getObjective(double[] beta) {
            return getGradient(beta);
        }
    };
    L_BFGS lbfgs = new L_BFGS().setGradEps(1e-12);
    L_BFGS.Result r = lbfgs.solve(gs, L_BFGS.startCoefs(2, 987654321));
    assertTrue("LBFGS failed to solve Rosenbrock function optimization", r.ginfo._objVal < 1e-4);
}

Example 4 with GradientSolver

use of hex.optimization.OptimizationUtils.GradientSolver in project h2o-3 by h2oai.

the class L_BFGS_Test method testArcene.

// Test LSM on arcene - wide dataset with ~10k columns
// test warm start and max #iteratoions
@Test
public void testArcene() {
    Key parsedKey = Key.make("arcene_parsed");
    DataInfo dinfo = null;
    try {
        Frame source = parse_test_file(parsedKey, "smalldata/glm_test/arcene.csv");
        Frame valid = new Frame(source._names.clone(), source.vecs().clone());
        GLMParameters glmp = new GLMParameters(Family.gaussian);
        glmp._lambda = new double[] { 1e-5 };
        glmp._alpha = new double[] { 0 };
        glmp._obj_reg = 0.01;
        dinfo = new DataInfo(source, valid, 1, false, DataInfo.TransformType.STANDARDIZE, DataInfo.TransformType.NONE, true, false, false, /* weights */
        false, /* offset */
        false, /* fold */
        false);
        DKV.put(dinfo._key, dinfo);
        GradientSolver solver = new GLMGradientSolver(null, glmp, dinfo, 1e-5, null);
        L_BFGS lbfgs = new L_BFGS().setMaxIter(20);
        double[] beta = MemoryManager.malloc8d(dinfo.fullN() + 1);
        beta[beta.length - 1] = new GLMWeightsFun(glmp).link(source.lastVec().mean());
        L_BFGS.Result r1 = lbfgs.solve(solver, beta.clone(), solver.getGradient(beta), new L_BFGS.ProgressMonitor() {

            int _i = 0;

            public boolean progress(double[] beta, GradientInfo ginfo) {
                System.out.println(++_i + ":" + ginfo._objVal);
                return true;
            }
        });
        lbfgs.setMaxIter(50);
        final int iter = r1.iter;
        L_BFGS.Result r2 = lbfgs.solve(solver, r1.coefs, r1.ginfo, new L_BFGS.ProgressMonitor() {

            int _i = 0;

            public boolean progress(double[] beta, GradientInfo ginfo) {
                System.out.println(iter + " + " + ++_i + ":" + ginfo._objVal);
                return true;
            }
        });
        System.out.println();
        lbfgs = new L_BFGS().setMaxIter(100);
        L_BFGS.Result r3 = lbfgs.solve(solver, beta.clone(), solver.getGradient(beta), new L_BFGS.ProgressMonitor() {

            int _i = 0;

            public boolean progress(double[] beta, GradientInfo ginfo) {
                System.out.println(++_i + ":" + ginfo._objVal + ", " + ArrayUtils.l2norm2(ginfo._gradient, false));
                return true;
            }
        });
        assertEquals(r1.iter, 20);
        //      assertEquals (r1.iter + r2.iter,r3.iter); // should be equal? got mismatch by 2
        assertEquals(r2.ginfo._objVal, r3.ginfo._objVal, 1e-8);
        assertEquals(.5 * glmp._lambda[0] * ArrayUtils.l2norm(r3.coefs, true) + r3.ginfo._objVal, 1e-4, 5e-4);
        assertTrue("iter# expected < 100, got " + r3.iter, r3.iter < 100);
    } finally {
        if (dinfo != null)
            DKV.remove(dinfo._key);
        Value v = DKV.get(parsedKey);
        if (v != null) {
            v.<Frame>get().delete();
        }
    }
}