Examples with AppendableVec water.fvec.AppendableVec used on opensource projects

Example 6 with AppendableVec

use of water.fvec.AppendableVec in project h2o-2 by h2oai.

the class NeuralNetMnist method csv.

private static void csv(String dest, String images, String labels) throws Exception {
    DataInputStream imagesBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(images))));
    DataInputStream labelsBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(labels))));
    // Magic
    imagesBuf.readInt();
    int count = imagesBuf.readInt();
    // Magic
    labelsBuf.readInt();
    assert count == labelsBuf.readInt();
    // Rows
    imagesBuf.readInt();
    // Cols
    imagesBuf.readInt();
    System.out.println("Count=" + count);
    byte[][] rawI = new byte[count][PIXELS];
    byte[] rawL = new byte[count];
    for (int n = 0; n < count; n++) {
        imagesBuf.readFully(rawI[n]);
        rawL[n] = labelsBuf.readByte();
    }
    MersenneTwisterRNG rand = new MersenneTwisterRNG(MersenneTwisterRNG.SEEDS);
    for (int n = count - 1; n >= 0; n--) {
        int shuffle = rand.nextInt(n + 1);
        byte[] image = rawI[shuffle];
        rawI[shuffle] = rawI[n];
        rawI[n] = image;
        byte label = rawL[shuffle];
        rawL[shuffle] = rawL[n];
        rawL[n] = label;
    }
    Vec[] vecs = new Vec[PIXELS + 1];
    NewChunk[] chunks = new NewChunk[vecs.length];
    for (int v = 0; v < vecs.length; v++) {
        vecs[v] = new AppendableVec(Key.make(UUID.randomUUID().toString()));
        chunks[v] = new NewChunk(vecs[v], 0);
    }
    for (int n = 0; n < count; n++) {
        for (int v = 0; v < vecs.length - 1; v++) chunks[v].addNum(rawI[n][v] & 0xff, 0);
        chunks[chunks.length - 1].addNum(rawL[n], 0);
    }
    for (int v = 0; v < vecs.length; v++) {
        chunks[v].close(0, null);
        vecs[v] = ((AppendableVec) vecs[v]).close(null);
    }
    Frame frame = new Frame(null, vecs);
    Utils.writeFileAndClose(new File(dest), frame.toCSV(false));
    imagesBuf.close();
    labelsBuf.close();
}

Example 7 with AppendableVec

use of water.fvec.AppendableVec in project h2o-3 by h2oai.

the class AstRecAssignTestUtils method seqStrVec.

static Vec seqStrVec(int... runs) {
    Key k = Vec.VectorGroup.VG_LEN1.addVec();
    Futures fs = new Futures();
    AppendableVec avec = new AppendableVec(k, Vec.T_STR);
    NewChunk chunk = new NewChunk(avec, 0);
    int seq = 0;
    for (int r : runs) {
        if (seq > 0)
            chunk.addStr(null);
        for (int i = 0; i < r; i++) chunk.addStr(String.valueOf(seq++));
    }
    chunk.close(0, fs);
    Vec vec = avec.layout_and_close(fs);
    fs.blockForPending();
    return vec;
}

Example 8 with AppendableVec

use of water.fvec.AppendableVec in project h2o-2 by h2oai.

the class ASTFunc method map.

@Override
double[] map(Env env, double[] in, double[] out) {
    final int sp = env._sp;
    Key key = Vec.VectorGroup.VG_LEN1.addVecs(1)[0];
    AppendableVec av = new AppendableVec(key);
    NewChunk nc = new NewChunk(av, 0);
    for (double v : in) nc.addNum(v);
    nc.close(0, null);
    Frame fr = new Frame(new String[] { "row" }, new Vec[] { av.close(null) });
    env.push(this);
    env.push(fr);
    this.apply(env, 2, null);
    if (env.isDbl()) {
        if (out == null || out.length < 1)
            out = new double[1];
        out[0] = env.popDbl();
    } else if (env.isAry()) {
        fr = env.peekAry();
        if (fr.vecs().length > 1)
            H2O.unimpl();
        Vec vec = fr.anyVec();
        if (vec.length() > 1 << 8)
            H2O.unimpl();
        if (out == null || out.length < vec.length())
            out = new double[(int) vec.length()];
        for (long i = 0; i < vec.length(); i++) out[(int) i] = vec.at(i);
        env.pop();
    } else {
        H2O.unimpl();
    }
    assert sp == env._sp;
    return out;
}