Examples with NewChunk water.fvec.NewChunk used on opensource projects

Example 26 with NewChunk

use of water.fvec.NewChunk in project h2o-3 by h2oai.

the class ChunkSplitter method extractChunkPart.

/** Extract portion of given chunk into given output chunk. */
public static void extractChunkPart(Chunk ic, Chunk oc, int startRow, int nrows, Futures fs) {
    try {
        NewChunk dst = new NewChunk(oc);
        dst._len = dst._sparseLen = 0;
        // Iterate over values skip all 0
        ic.extractRows(dst, startRow, startRow + nrows);
        // Handle case when last added value is followed by zeros till startRow+nrows
        assert dst._len == oc._len : "NewChunk.dst.len = " + dst._len + ", oc._len = " + oc._len;
        dst.close(dst.cidx(), fs);
    } catch (RuntimeException t) {
        Log.err("got exception in chunkSplitter, ic = " + ic + ", oc = " + oc + " startRow = " + startRow + " nrows = " + nrows);
        throw t;
    }
}

Example 27 with NewChunk

use of water.fvec.NewChunk in project h2o-3 by h2oai.

the class VecUtils method UUIDToStringVec.

/**
   * Create a new {@link Vec} of string values from a UUID {@link Vec}.
   *
   * String {@link Vec} is the standard hexadecimal representations of a UUID.
   *
   * @param src a UUID {@link Vec}
   * @return a string {@link Vec}
   */
public static Vec UUIDToStringVec(Vec src) {
    if (!src.isUUID())
        throw new H2OIllegalArgumentException("UUIDToStringVec() conversion only works on UUID columns");
    Vec res = new MRTask() {

        @Override
        public void map(Chunk chk, NewChunk newChk) {
            if (chk instanceof C0DChunk) {
                // all NAs
                for (int i = 0; i < chk._len; i++) newChk.addNA();
            } else {
                for (int i = 0; i < chk._len; i++) {
                    if (!chk.isNA(i))
                        newChk.addStr(PrettyPrint.UUID(chk.at16l(i), chk.at16h(i)));
                    else
                        newChk.addNA();
                }
            }
        }
    }.doAll(Vec.T_STR, src).outputFrame().anyVec();
    assert res != null;
    return res;
}

Example 28 with NewChunk

use of water.fvec.NewChunk in project h2o-2 by h2oai.

the class NeuralNetMnist method csv.

private static void csv(String dest, String images, String labels) throws Exception {
    DataInputStream imagesBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(images))));
    DataInputStream labelsBuf = new DataInputStream(new GZIPInputStream(new FileInputStream(new File(labels))));
    // Magic
    imagesBuf.readInt();
    int count = imagesBuf.readInt();
    // Magic
    labelsBuf.readInt();
    assert count == labelsBuf.readInt();
    // Rows
    imagesBuf.readInt();
    // Cols
    imagesBuf.readInt();
    System.out.println("Count=" + count);
    byte[][] rawI = new byte[count][PIXELS];
    byte[] rawL = new byte[count];
    for (int n = 0; n < count; n++) {
        imagesBuf.readFully(rawI[n]);
        rawL[n] = labelsBuf.readByte();
    }
    MersenneTwisterRNG rand = new MersenneTwisterRNG(MersenneTwisterRNG.SEEDS);
    for (int n = count - 1; n >= 0; n--) {
        int shuffle = rand.nextInt(n + 1);
        byte[] image = rawI[shuffle];
        rawI[shuffle] = rawI[n];
        rawI[n] = image;
        byte label = rawL[shuffle];
        rawL[shuffle] = rawL[n];
        rawL[n] = label;
    }
    Vec[] vecs = new Vec[PIXELS + 1];
    NewChunk[] chunks = new NewChunk[vecs.length];
    for (int v = 0; v < vecs.length; v++) {
        vecs[v] = new AppendableVec(Key.make(UUID.randomUUID().toString()));
        chunks[v] = new NewChunk(vecs[v], 0);
    }
    for (int n = 0; n < count; n++) {
        for (int v = 0; v < vecs.length - 1; v++) chunks[v].addNum(rawI[n][v] & 0xff, 0);
        chunks[chunks.length - 1].addNum(rawL[n], 0);
    }
    for (int v = 0; v < vecs.length; v++) {
        chunks[v].close(0, null);
        vecs[v] = ((AppendableVec) vecs[v]).close(null);
    }
    Frame frame = new Frame(null, vecs);
    Utils.writeFileAndClose(new File(dest), frame.toCSV(false));
    imagesBuf.close();
    labelsBuf.close();
}

Example 29 with NewChunk

use of water.fvec.NewChunk in project h2o-2 by h2oai.

the class MRUtils method div.

public static Frame div(Frame fr, final double d) {
    Frame r = new MRTask2() {

        @Override
        public void map(Chunk[] cs, NewChunk[] ncs) {
            for (int i = 0; i < ncs.length; i++) {
                NewChunk nc = ncs[i];
                Chunk c = cs[i];
                for (int r = 0; r < c._len; r++) if (d != 0)
                    nc.addNum(c.at0(r) * d);
                else
                    nc.addNA();
            }
        }
    }.doAll(fr.numCols(), fr).outputFrame(fr.names(), fr.domains());
    return r;
}

Example 30 with NewChunk

use of water.fvec.NewChunk in project h2o-3 by h2oai.

the class AstRecAssignTestUtils method seqStrVec.

static Vec seqStrVec(int... runs) {
    Key k = Vec.VectorGroup.VG_LEN1.addVec();
    Futures fs = new Futures();
    AppendableVec avec = new AppendableVec(k, Vec.T_STR);
    NewChunk chunk = new NewChunk(avec, 0);
    int seq = 0;
    for (int r : runs) {
        if (seq > 0)
            chunk.addStr(null);
        for (int i = 0; i < r; i++) chunk.addStr(String.valueOf(seq++));
    }
    chunk.close(0, fs);
    Vec vec = avec.layout_and_close(fs);
    fs.blockForPending();
    return vec;
}