Examples with Futures water.Futures used on opensource projects

Example 16 with Futures

use of water.Futures in project h2o-2 by h2oai.

the class Env method subRef.

// Lower the refcnt on all vecs in this frame.
// Immediately free all vecs with zero count.
// Always return a null.
public Frame subRef(Frame fr, String key) {
    if (fr == null)
        return null;
    Futures fs = new Futures();
    for (Vec vec : fr.vecs()) subRef(vec, fs);
    fs.blockForPending();
    return null;
}

Example 17 with Futures

use of water.Futures in project h2o-3 by h2oai.

the class FrameV3 method fillFromImpl.

public FrameV3 fillFromImpl(Frame f, long row_offset, int row_count, int column_offset, int column_count) {
    // 100 rows by default
    if (row_count == 0)
        row_count = 100;
    // full width by default
    if (column_count == 0)
        column_count = f.numCols() - column_offset;
    row_count = (int) Math.min(row_count, row_offset + f.numRows());
    column_count = Math.min(column_count, column_offset + f.numCols());
    this.frame_id = new FrameKeyV3(f._key);
    this.checksum = f.checksum();
    this.byte_size = f.byteSize();
    this.row_offset = row_offset;
    this.rows = f.numRows();
    this.num_columns = f.numCols();
    this.row_count = row_count;
    this.total_column_count = f.numCols();
    this.column_offset = column_offset;
    this.column_count = column_count;
    this.columns = new ColV3[column_count];
    Vec[] vecs = f.vecs();
    Futures fs = new Futures();
    // NOTE: SKIP deleted Vecs!  The columns entry will be null for deleted Vecs.
    for (int i = 0; i < column_count; i++) if (null == DKV.get(vecs[column_offset + i]._key))
        Log.warn("For Frame: " + f._key + ", Vec number: " + (column_offset + i) + " (" + f.name((column_offset + i)) + ") is missing; not returning it.");
    else
        vecs[column_offset + i].startRollupStats(fs);
    for (int i = 0; i < column_count; i++) if (null == DKV.get(vecs[column_offset + i]._key))
        Log.warn("For Frame: " + f._key + ", Vec number: " + (column_offset + i) + " (" + f.name((column_offset + i)) + ") is missing; not returning it.");
    else
        columns[i] = new ColV3(f._names[column_offset + i], vecs[column_offset + i], this.row_offset, this.row_count);
    fs.blockForPending();
    this.is_text = f.numCols() == 1 && vecs[0] instanceof ByteVec;
    this.default_percentiles = Vec.PERCENTILES;
    ChunkSummary cs = FrameUtils.chunkSummary(f);
    this.chunk_summary = new TwoDimTableV3(cs.toTwoDimTableChunkTypes());
    this.distribution_summary = new TwoDimTableV3(cs.toTwoDimTableDistribution());
    this._fr = f;
    return this;
}

Example 18 with Futures

use of water.Futures in project h2o-3 by h2oai.

the class VecStatsTest method test.

@Test
public void test() {
    Frame frame = null;
    try {
        Futures fs = new Futures();
        Random random = new Random();
        Vec[] vecs = new Vec[1];
        AppendableVec vec = new AppendableVec(Vec.newKey(), Vec.T_NUM);
        for (int i = 0; i < 2; i++) {
            NewChunk chunk = new NewChunk(vec, i);
            for (int r = 0; r < 1000; r++) chunk.addNum(random.nextInt(1000));
            chunk.close(i, fs);
        }
        vecs[0] = vec.layout_and_close(fs);
        fs.blockForPending();
        frame = new Frame(Key.<Frame>make(), null, vecs);
        // Make sure we test the multi-chunk case
        vecs = frame.vecs();
        assert vecs[0].nChunks() > 1;
        long rows = frame.numRows();
        Vec v = vecs[0];
        double min = Double.POSITIVE_INFINITY, max = Double.NEGATIVE_INFINITY, mean = 0, sigma = 0;
        for (int r = 0; r < rows; r++) {
            double d = v.at(r);
            if (d < min)
                min = d;
            if (d > max)
                max = d;
            mean += d;
        }
        mean /= rows;
        for (int r = 0; r < rows; r++) {
            double d = v.at(r);
            sigma += (d - mean) * (d - mean);
        }
        sigma = Math.sqrt(sigma / (rows - 1));
        double epsilon = 1e-9;
        assertEquals(max, v.max(), epsilon);
        assertEquals(min, v.min(), epsilon);
        assertEquals(mean, v.mean(), epsilon);
        assertEquals(sigma, v.sigma(), epsilon);
    } finally {
        if (frame != null)
            frame.delete();
    }
}

Example 19 with Futures

use of water.Futures in project h2o-3 by h2oai.

the class NewChunkTest method post.

private void post() {
    cc = nc.compress();
    //HACK
    av._tmp_espc[0] = K;
    //HACK
    cc._start = 0;
    // HACK as well
    cc._cidx = 0;
    Futures fs = new Futures();
    vec = cc._vec = av.layout_and_close(fs);
    fs.blockForPending();
    //only the vec header is in DKV, the chunk is not
    assert (DKV.get(vec._key) != null);
}

Example 20 with Futures

use of water.Futures in project h2o-3 by h2oai.

the class C4ChunkTest method test_setNA.

@Test
public void test_setNA() {
    // Create a vec with one chunk with 15 elements, and set its numbers
    water.Key key = Vec.newKey();
    Vec vec = new Vec(key, Vec.ESPC.rowLayout(key, new long[] { 0, 15 })).makeZero();
    int[] vals = new int[] { 0, 3, 0, 6, 0, 0, 0, Integer.MIN_VALUE + 1, 0, 12, Integer.MAX_VALUE, 32767, 0, 0, 19 };
    Vec.Writer w = vec.open();
    for (int i = 0; i < vals.length; ++i) w.set(i, vals[i]);
    w.close();
    Chunk cc = vec.chunkForChunkIdx(0);
    assert cc instanceof C4Chunk;
    Futures fs = new Futures();
    fs.blockForPending();
    for (int i = 0; i < vals.length; ++i) Assert.assertEquals(vals[i], cc.at8(i));
    for (int i = 0; i < vals.length; ++i) Assert.assertEquals(vals[i], cc.at8_abs(i));
    int[] NAs = new int[] { 1, 5, 2 };
    int[] notNAs = new int[] { 0, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14 };
    for (int na : NAs) cc.setNA_abs(na);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    NewChunk nc = new NewChunk(null, 0);
    cc.extractRows(nc, 0, (int) vec.length());
    Assert.assertEquals(vals.length, nc._sparseLen);
    Assert.assertEquals(vals.length, nc._len);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    Chunk cc2 = nc.compress();
    Assert.assertEquals(vals.length, cc._len);
    Assert.assertTrue(cc2 instanceof C4Chunk);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    Assert.assertTrue(Arrays.equals(cc._mem, cc2._mem));
    vec.remove();
}