Examples with MRTask water.MRTask used on opensource projects

Example 21 with MRTask

use of water.MRTask in project h2o-3 by h2oai.

the class AstBinOp method frame_op_scalar.

/**
   * Auto-widen the scalar to every element of the frame
   */
public ValFrame frame_op_scalar(Frame fr, final double d) {
    Frame res = new MRTask() {

        @Override
        public void map(Chunk[] chks, NewChunk[] cress) {
            for (int c = 0; c < chks.length; c++) {
                Chunk chk = chks[c];
                NewChunk cres = cress[c];
                for (int i = 0; i < chk._len; i++) cres.addNum(op(chk.atd(i), d));
            }
        }
    }.doAll(fr.numCols(), Vec.T_NUM, fr).outputFrame(fr._names, null);
    // Cleanup categorical misuse
    return cleanCategorical(fr, res);
}

Example 22 with MRTask

use of water.MRTask in project h2o-3 by h2oai.

the class AstCumu method apply.

@Override
public ValFrame apply(Env env, Env.StackHelp stk, AstRoot[] asts) {
    Frame f = stk.track(asts[1].exec(env)).getFrame();
    AstRoot axisAR = asts[2];
    for (Vec v : f.vecs()) {
        if (v.isCategorical() || v.isString() || v.isUUID())
            throw new IllegalArgumentException("Cumulative functions not applicable to enum, string, or UUID values");
    }
    double axis = axisAR.exec(env).getNum();
    if (axis != 1.0 && axis != 0.0)
        throw new IllegalArgumentException("Axis must be 0 or 1");
    if (f.numCols() == 1) {
        if (axis == 0.0) {
            AstCumu.CumuTask t = new AstCumu.CumuTask(f.anyVec().nChunks(), init());
            t.doAll(new byte[] { Vec.T_NUM }, f.anyVec());
            final double[] chkCumu = t._chkCumu;
            Vec cumuVec = t.outputFrame().anyVec();
            new MRTask() {

                @Override
                public void map(Chunk c) {
                    if (c.cidx() != 0) {
                        double d = chkCumu[c.cidx() - 1];
                        for (int i = 0; i < c._len; ++i) c.set(i, op(c.atd(i), d));
                    }
                }
            }.doAll(cumuVec);
            Key<Frame> k = Key.make();
            return new ValFrame(new Frame(k, null, new Vec[] { cumuVec }));
        } else {
            return new ValFrame(new Frame(f));
        }
    } else {
        if (axis == 0.0) {
            // down the column implementation
            AstCumu.CumuTaskWholeFrame t = new AstCumu.CumuTaskWholeFrame(f.anyVec().nChunks(), init(), f.numCols());
            Frame fr2 = t.doAll(f.numCols(), Vec.T_NUM, f).outputFrame(null, f.names(), null);
            final double[][] chkCumu = t._chkCumu;
            new MRTask() {

                @Override
                public void map(Chunk[] cs) {
                    if (cs[0].cidx() != 0) {
                        for (int i = 0; i < cs.length; i++) {
                            double d = chkCumu[i][cs[i].cidx() - 1];
                            for (int j = 0; j < cs[i]._len; ++j) cs[i].set(j, op(cs[i].atd(j), d));
                        }
                    }
                }
            }.doAll(fr2);
            return new ValFrame(new Frame(fr2));
        } else {
            AstCumu.CumuTaskAxis1 t = new AstCumu.CumuTaskAxis1(init());
            Frame fr2 = t.doAll(f.numCols(), Vec.T_NUM, f).outputFrame(null, f.names(), null);
            return new ValFrame(new Frame(fr2));
        }
    }
}

Example 23 with MRTask

use of water.MRTask in project h2o-3 by h2oai.

the class AstRectangleConditionalAssignTest method makeTestFrame.

private Frame makeTestFrame() {
    Frame fr = null;
    Vec v = ivec(1, 2, 3, 4, 5);
    try {
        fr = new MRTask() {

            @Override
            public void map(Chunk[] cs, NewChunk[] ncs) {
                for (int i = 0; i < cs[0]._len; i++) {
                    int r = (int) cs[0].atd(i);
                    ncs[0].addNum(r);
                    ncs[1].addNum(11.2 * r);
                    ncs[2].addUUID(r, r * 10);
                    ncs[3].addStr("row" + r);
                    ncs[4].addCategorical(r % 2 == 0 ? 0 : 1);
                }
            }
        }.doAll(new byte[] { Vec.T_NUM, Vec.T_NUM, Vec.T_UUID, Vec.T_STR, Vec.T_CAT }, v).outputFrame(Key.make("data"), new String[] { "v1", "v2", "v3", "v4", "v5" }, new String[][] { null, null, null, null, new String[] { "a", "b" } });
    } finally {
        v.remove();
    }
    assert fr != null;
    return fr;
}

Example 24 with MRTask

use of water.MRTask in project h2o-3 by h2oai.

the class ShuffleSplitFrameTest method testShuffleSplitWithMultipleColumns.

@Test
public /* this test makes sure that the rows of the split frames are preserved (including UUID) */
void testShuffleSplitWithMultipleColumns() {
    long[] chunkLayout = ar(2L, 2L, 3L);
    String[][] data = ar(ar("1", "2"), ar(null, "3"), ar("4", "5", "6"));
    Frame f = null;
    Frame tmpFrm = createFrame("test1.hex", chunkLayout, data);
    try {
        f = new MRTask() {

            @Override
            public void map(Chunk[] cs, NewChunk[] ncs) {
                for (int i = 0; i < cs[0]._len; i++) {
                    BufferedString bs = cs[0].atStr(new BufferedString(), i);
                    int val = bs == null ? 0 : Integer.parseInt(bs.toString());
                    ncs[0].addStr(bs);
                    ncs[1].addNum(val);
                    ncs[2].addNum(i);
                    ncs[3].addUUID(i, val);
                }
            }
        }.doAll(new byte[] { Vec.T_STR, Vec.T_NUM, Vec.T_NUM, Vec.T_UUID }, tmpFrm).outputFrame();
    } finally {
        tmpFrm.delete();
    }
    testScenario(f, flat(data), new MRTask() {

        @Override
        public void map(Chunk[] cs) {
            for (int i = 0; i < cs[0]._len; i++) {
                BufferedString bs = cs[0].atStr(new BufferedString(), i);
                int expectedVal = bs == null ? 0 : Integer.parseInt(bs.toString());
                int expectedIndex = (int) cs[2].atd(i);
                Assert.assertEquals((double) expectedVal, cs[1].atd(i), 0.00001);
                Assert.assertEquals(expectedIndex, (int) cs[3].at16l(i));
                Assert.assertEquals(expectedVal, (int) cs[3].at16h(i));
            }
        }
    });
}

Example 25 with MRTask

use of water.MRTask in project h2o-3 by h2oai.

the class Session method cluster_init.

static void cluster_init() {
    if (_initialized)
        return;
    // Touch a common class to force loading
    new MRTask() {

        @Override
        public void setupLocal() {
            new AstPlus();
        }
    }.doAllNodes();
    _initialized = true;
}