Examples with AppendableVec water.fvec.AppendableVec used on opensource projects

Example 1 with AppendableVec

use of water.fvec.AppendableVec in project h2o-3 by h2oai.

the class SVMLightFVecParseWriter method addColumns.

private void addColumns(int newColCnt) {
    int oldColCnt = _vecs.length;
    if (newColCnt > oldColCnt) {
        _nvs = Arrays.copyOf(_nvs, newColCnt);
        _vecs = Arrays.copyOf(_vecs, newColCnt);
        for (int i = oldColCnt; i < newColCnt; ++i) {
            _vecs[i] = new AppendableVec(_vg.vecKey(i + _vecIdStart), _vecs[0]._tmp_espc, Vec.T_NUM, _vecs[0]._chunkOff);
            _nvs[i] = new NewChunk(_vecs[i], _cidx, true);
        }
        _nCols = newColCnt;
    }
}

Example 2 with AppendableVec

use of water.fvec.AppendableVec in project h2o-3 by h2oai.

the class AstLs method apply.

@Override
public ValFrame apply(Env env, Env.StackHelp stk, AstRoot[] asts) {
    ArrayList<String> domain = new ArrayList<>();
    Futures fs = new Futures();
    AppendableVec av = new AppendableVec(Vec.VectorGroup.VG_LEN1.addVec(), Vec.T_CAT);
    NewChunk keys = new NewChunk(av, 0);
    int r = 0;
    for (Key key : KeySnapshot.globalSnapshot().keys()) {
        keys.addCategorical(r++);
        domain.add(key.toString());
    }
    String[] key_domain = domain.toArray(new String[domain.size()]);
    av.setDomain(key_domain);
    keys.close(fs);
    // c0 is the row index vec
    Vec c0 = av.layout_and_close(fs);
    fs.blockForPending();
    return new ValFrame(new Frame(Key.<Frame>make("h2o_ls"), new String[] { "key" }, new Vec[] { c0 }));
}

Example 3 with AppendableVec

use of water.fvec.AppendableVec in project h2o-3 by h2oai.

the class ArrayUtils method frame.

/** Create a new frame based on given row data.
   *  @param key   Key for the frame
   *  @param names names of frame columns
   *  @param rows  data given in the form of rows
   *  @return new frame which contains columns named according given names and including given data */
public static Frame frame(Key<Frame> key, String[] names, double[]... rows) {
    assert names == null || names.length == rows[0].length;
    Futures fs = new Futures();
    Vec[] vecs = new Vec[rows[0].length];
    Key<Vec>[] keys = Vec.VectorGroup.VG_LEN1.addVecs(vecs.length);
    int rowLayout = -1;
    for (int c = 0; c < vecs.length; c++) {
        AppendableVec vec = new AppendableVec(keys[c], Vec.T_NUM);
        NewChunk chunk = new NewChunk(vec, 0);
        for (double[] row : rows) chunk.addNum(row[c]);
        chunk.close(0, fs);
        if (rowLayout == -1)
            rowLayout = vec.compute_rowLayout();
        vecs[c] = vec.close(rowLayout, fs);
    }
    fs.blockForPending();
    Frame fr = new Frame(key, names, vecs);
    if (key != null)
        DKV.put(key, fr);
    return fr;
}

Example 4 with AppendableVec

use of water.fvec.AppendableVec in project h2o-2 by h2oai.

the class ASTFunc method map.

@Override
double[] map(Env env, double[] in, double[] out) {
    final int sp = env._sp;
    Key key = Vec.VectorGroup.VG_LEN1.addVecs(1)[0];
    AppendableVec av = new AppendableVec(key);
    NewChunk nc = new NewChunk(av, 0);
    for (double v : in) nc.addNum(v);
    nc.close(0, null);
    Frame fr = new Frame(new String[] { "row" }, new Vec[] { av.close(null) });
    env.push(this);
    env.push(fr);
    this.apply(env, 2, null);
    if (env.isDbl()) {
        if (out == null || out.length < 1)
            out = new double[1];
        out[0] = env.popDbl();
    } else if (env.isAry()) {
        fr = env.peekAry();
        if (fr.vecs().length > 1)
            H2O.unimpl();
        Vec vec = fr.anyVec();
        if (vec.length() > 1 << 8)
            H2O.unimpl();
        if (out == null || out.length < vec.length())
            out = new double[(int) vec.length()];
        for (long i = 0; i < vec.length(); i++) out[(int) i] = vec.at(i);
        env.pop();
    } else {
        H2O.unimpl();
    }
    assert sp == env._sp;
    return out;
}

Example 5 with AppendableVec

use of water.fvec.AppendableVec in project h2o-2 by h2oai.

the class NeuralNetMnist method csv.

private static void csv(String dest, String images, String labels) throws Exception {
    DataInputStream imagesBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(images))));
    DataInputStream labelsBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(labels))));
    // Magic
    imagesBuf.readInt();
    int count = imagesBuf.readInt();
    // Magic
    labelsBuf.readInt();
    assert count == labelsBuf.readInt();
    // Rows
    imagesBuf.readInt();
    // Cols
    imagesBuf.readInt();
    System.out.println("Count=" + count);
    byte[][] rawI = new byte[count][PIXELS];
    byte[] rawL = new byte[count];
    for (int n = 0; n < count; n++) {
        imagesBuf.readFully(rawI[n]);
        rawL[n] = labelsBuf.readByte();
    }
    MersenneTwisterRNG rand = new MersenneTwisterRNG(MersenneTwisterRNG.SEEDS);
    for (int n = count - 1; n >= 0; n--) {
        int shuffle = rand.nextInt(n + 1);
        byte[] image = rawI[shuffle];
        rawI[shuffle] = rawI[n];
        rawI[n] = image;
        byte label = rawL[shuffle];
        rawL[shuffle] = rawL[n];
        rawL[n] = label;
    }
    Vec[] vecs = new Vec[PIXELS + 1];
    NewChunk[] chunks = new NewChunk[vecs.length];
    for (int v = 0; v < vecs.length; v++) {
        vecs[v] = new AppendableVec(Key.make(UUID.randomUUID().toString()));
        chunks[v] = new NewChunk(vecs[v], 0);
    }
    for (int n = 0; n < count; n++) {
        for (int v = 0; v < vecs.length - 1; v++) chunks[v].addNum(rawI[n][v] & 0xff, 0);
        chunks[chunks.length - 1].addNum(rawL[n], 0);
    }
    for (int v = 0; v < vecs.length; v++) {
        chunks[v].close(0, null);
        vecs[v] = ((AppendableVec) vecs[v]).close(null);
    }
    Frame frame = new Frame(null, vecs);
    Utils.writeFileAndClose(new File(dest), frame.toCSV(false));
    imagesBuf.close();
    labelsBuf.close();
}