Examples with Job water.Job used on opensource projects

Example 11 with Job

use of water.Job in project h2o-3 by h2oai.

the class QuantileTest method testWeights1.

@Test
public void testWeights1() {
    QuantileModel kmm1;
    QuantileModel kmm2;
    Frame fr1 = null;
    Frame fr2 = null;
    try {
        fr1 = parse_test_file("smalldata/junit/no_weights.csv");
        fr2 = parse_test_file("smalldata/junit/weights.csv");
        for (QuantileModel.CombineMethod comb : new QuantileModel.CombineMethod[] { QuantileModel.CombineMethod.AVERAGE, QuantileModel.CombineMethod.LOW, QuantileModel.CombineMethod.HIGH, QuantileModel.CombineMethod.INTERPOLATE }) {
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr1._key;
                parms._combine_method = comb;
                parms._weights_column = null;
                Job<QuantileModel> job1 = new Quantile(parms).trainModel();
                kmm1 = job1.get();
                job1.remove();
            }
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr2._key;
                parms._combine_method = comb;
                parms._weights_column = "weight";
                Job<QuantileModel> job2 = new Quantile(parms).trainModel();
                kmm2 = job2.get();
                job2.remove();
            }
            try {
                Assert.assertTrue(Arrays.deepEquals(kmm1._output._quantiles, kmm2._output._quantiles));
            } finally {
                if (kmm1 != null)
                    kmm1.delete();
                if (kmm2 != null)
                    kmm2.delete();
            }
        }
    } finally {
        if (fr1 != null)
            fr1.remove();
        if (fr2 != null)
            fr2.remove();
    }
}

Example 12 with Job

use of water.Job in project h2o-3 by h2oai.

the class QuantileTest method testWeights0.

@Test
public void testWeights0() {
    QuantileModel kmm1;
    QuantileModel kmm2;
    Frame fr1 = null;
    Frame fr2 = null;
    try {
        fr1 = parse_test_file("smalldata/junit/no_weights.csv");
        fr2 = parse_test_file("smalldata/junit/weights_all_ones.csv");
        for (QuantileModel.CombineMethod comb : new QuantileModel.CombineMethod[] { QuantileModel.CombineMethod.AVERAGE, QuantileModel.CombineMethod.LOW, QuantileModel.CombineMethod.HIGH, QuantileModel.CombineMethod.INTERPOLATE }) {
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr1._key;
                parms._combine_method = comb;
                parms._weights_column = null;
                Job<QuantileModel> job1 = new Quantile(parms).trainModel();
                kmm1 = job1.get();
                job1.remove();
            }
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr2._key;
                parms._combine_method = comb;
                parms._weights_column = "weight";
                Job<QuantileModel> job2 = new Quantile(parms).trainModel();
                kmm2 = job2.get();
                job2.remove();
            }
            try {
                Assert.assertTrue(Arrays.deepEquals(kmm1._output._quantiles, kmm2._output._quantiles));
            } finally {
                if (kmm1 != null)
                    kmm1.delete();
                if (kmm2 != null)
                    kmm2.delete();
            }
        }
    } finally {
        if (fr1 != null)
            fr1.remove();
        if (fr2 != null)
            fr2.remove();
    }
}

Example 13 with Job

use of water.Job in project h2o-3 by h2oai.

the class QuantileTest method testShuffled.

@Test
public void testShuffled() {
    QuantileModel kmm1;
    QuantileModel kmm2;
    Frame fr1 = null;
    Frame fr2 = null;
    try {
        fr1 = parse_test_file("smalldata/junit/no_weights.csv");
        fr2 = parse_test_file("smalldata/junit/no_weights_shuffled.csv");
        for (QuantileModel.CombineMethod comb : new QuantileModel.CombineMethod[] { QuantileModel.CombineMethod.AVERAGE, QuantileModel.CombineMethod.LOW, QuantileModel.CombineMethod.HIGH, QuantileModel.CombineMethod.INTERPOLATE }) {
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr1._key;
                parms._combine_method = comb;
                Job<QuantileModel> job1 = new Quantile(parms).trainModel();
                kmm1 = job1.get();
                job1.remove();
            }
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr2._key;
                parms._combine_method = comb;
                Job<QuantileModel> job2 = new Quantile(parms).trainModel();
                kmm2 = job2.get();
                job2.remove();
            }
            try {
                Assert.assertTrue(Arrays.deepEquals(kmm1._output._quantiles, kmm2._output._quantiles));
            } finally {
                if (kmm1 != null)
                    kmm1.delete();
                if (kmm2 != null)
                    kmm2.delete();
            }
        }
    } finally {
        if (fr1 != null)
            fr1.remove();
        if (fr2 != null)
            fr2.remove();
    }
}

Example 14 with Job

use of water.Job in project h2o-3 by h2oai.

the class AstQtile method apply.

@Override
public ValFrame apply(Env env, Env.StackHelp stk, AstRoot[] asts) {
    QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
    Frame fr = stk.track(asts[1].exec(env)).getFrame();
    // Force a bogus Key for Quantiles ModelBuilder
    Frame fr_wkey = new Frame(fr);
    DKV.put(fr_wkey);
    parms._train = fr_wkey._key;
    parms._probs = ((AstNumList) asts[2]).expand();
    for (double d : parms._probs) if (d < 0 || d > 1)
        throw new IllegalArgumentException("Probability must be between 0 and 1: " + d);
    String inter = asts[3].exec(env).getStr();
    parms._combine_method = QuantileModel.CombineMethod.valueOf(inter.toUpperCase());
    parms._weights_column = asts[4].str().equals("_") ? null : asts[4].str();
    // Compute Quantiles
    Job j = new Quantile(parms).trainModel();
    QuantileModel q = (QuantileModel) j.get();
    DKV.remove(j._key);
    // Remove bogus Key
    DKV.remove(fr_wkey._key);
    // Reshape all outputs as a Frame, with probs in col 0 and the
    // quantiles in cols 1 thru fr.numCols() - except the optional weights vec
    int ncols = fr.numCols();
    if (parms._weights_column != null)
        ncols--;
    Vec[] vecs = new Vec[1 + /*1 more for the probs themselves*/
    ncols];
    String[] names = new String[vecs.length];
    vecs[0] = Vec.makeCon(null, parms._probs);
    names[0] = "Probs";
    int w = 0;
    for (int i = 0; i < vecs.length - 1; ++i) {
        if (fr._names[i].equals(parms._weights_column))
            w = 1;
        vecs[i + 1] = Vec.makeCon(null, q._output._quantiles[i]);
        names[i + 1] = fr._names[w + i] + "Quantiles";
    }
    q.delete();
    return new ValFrame(new Frame(names, vecs));
}

Example 15 with Job

use of water.Job in project h2o-3 by h2oai.

the class QuantileTest method testWeights2.

@Ignore
@Test
public void testWeights2() {
    //same behavior as wtd.quantile in R -> results with all weights=1 and all weights=2 don't agree (unless normwt=TRUE)
    QuantileModel kmm1;
    QuantileModel kmm2;
    Frame fr1 = null;
    Frame fr2 = null;
    try {
        fr1 = parse_test_file("smalldata/junit/weights_all_twos.csv");
        fr2 = parse_test_file("smalldata/junit/weights_all_ones.csv");
        for (QuantileModel.CombineMethod comb : new QuantileModel.CombineMethod[] { QuantileModel.CombineMethod.AVERAGE, QuantileModel.CombineMethod.LOW, QuantileModel.CombineMethod.HIGH, QuantileModel.CombineMethod.INTERPOLATE }) {
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr1._key;
                parms._combine_method = comb;
                parms._weights_column = "weight";
                Job<QuantileModel> job1 = new Quantile(parms).trainModel();
                kmm1 = job1.get();
                job1.remove();
            }
            {
                QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
                parms._train = fr2._key;
                parms._combine_method = comb;
                parms._weights_column = "weight";
                Job<QuantileModel> job2 = new Quantile(parms).trainModel();
                kmm2 = job2.get();
                job2.remove();
            }
            try {
                assert (Arrays.deepEquals(kmm1._output._quantiles, kmm2._output._quantiles));
                Assert.assertTrue(Arrays.deepEquals(kmm1._output._quantiles, kmm2._output._quantiles));
            } finally {
                if (kmm1 != null)
                    kmm1.delete();
                if (kmm2 != null)
                    kmm2.delete();
            }
        }
    } finally {
        if (fr1 != null)
            fr1.remove();
        if (fr2 != null)
            fr2.remove();
    }
}