Examples with Vec water.fvec.Vec used on opensource projects

Example 1 with Vec

use of water.fvec.Vec in project h2o-2 by h2oai.

the class ChunkDemo method Chunk.

@Test
public void Chunk() {
    String fileName = "./cookbookData/iris_withNA.csv";
    File file = new File(fileName);
    Key fkey = NFSFileVec.make(file);
    Key okey = Key.make("iris.hex");
    Frame fr;
    fr = ParseDataset2.parse(okey, new Key[] { fkey });
    //accessing the first vector from the frame
    Vec vv = fr.vec(0);
    int chunk_num = vv.nChunks();
    System.out.println("Number of chunks in column 1:  " + chunk_num);
    //Reading in the first chunk. This loads the data locally.
    Chunk cc = vv.chunkForChunkIdx(0);
    for (int i = 0; i < cc._len; i++) {
        //READING A DOUBLE ELEMENT FROM A CHUNK
        // at0 gives the local chunk index
        double d_at = cc.at0(i);
        System.out.println("double Value at chunk index " + i + ":  " + d_at);
        //READING A LONG ELEMENT FROM A CHUNK
        if (!Double.isNaN(d_at)) {
            long l_at = cc.at80(i);
            System.out.println("long Value at chunk index " + i + ":  " + l_at);
        }
        //UPDATING A DOUBLE ELEMENT TO A CHUNK
        double d = 1.23;
        double set_dval = cc.set0(i, d);
        System.out.println("Setting a double value at index " + i + " : " + set_dval);
        //UPDATING A LONG ELEMENT TO A CHUNK
        long l = 123L;
        long set_lval = cc.set0(i, l);
        System.out.println("Setting a double value at index " + i + " : " + set_lval);
    }
    //logThisH2OInstanceWebBrowserAddress();
    //sleepForever();	
    //CLEANING THE KV STORE OF ALL DATA
    Frame.delete(okey);
}

Example 2 with Vec

use of water.fvec.Vec in project h2o-2 by h2oai.

the class FillNAsWithMeanDemo01 method frame_001.

@Test
public void frame_001() {
    //String fileName = "/Users/nidhimehta/h2o/smalldata/iris/iris.csv";
    //String fileName = "/Users/nidhimehta/Desktop/data/covtype/covtrain_tit";
    //String fileName = "/Users/nidhimehta/Desktop/iris_withNA.csv";
    String fileName = "./cookbookData/iris_withNA.csv";
    File file = new File(fileName);
    Key fkey = NFSFileVec.make(file);
    Key okey = Key.make("iris.hex");
    Frame fr;
    fr = ParseDataset2.parse(okey, new Key[] { fkey });
    int len = fr.numCols();
    for (int i = 0; i < len; i++) {
        Vec vv = fr.vec(i);
        // creating a new vector same as original vector filled with zeros 
        Vec output = vv.makeZero();
        // map reduce call
        FillNasWithMean lr1 = new FillNasWithMean(vv.mean()).doAll(vv, output);
        // adding the vector to the original frame
        fr.add("FilledNa" + i, output);
    }
    Log.info("frame              : " + fr);
    //logThisH2OInstanceWebBrowserAddress();
    //sleepForever();
    Frame.delete(okey);
}

Example 3 with Vec

use of water.fvec.Vec in project h2o-2 by h2oai.

the class FillNAsWithMeanDemo03 method frame_001.

@Test
public void frame_001() {
    // String fileName = "/Users/nidhimehta/h2o/smalldata/iris/iris.csv";
    //String fileName = "/Users/nidhimehta/Desktop/data/covtype/covtrain_tit";
    //String fileName = "/Users/nidhimehta/Desktop/iris_withNA.csv";
    String fileName = "./cookbookData/iris_withNA.csv";
    File file = new File(fileName);
    Key fkey = NFSFileVec.make(file);
    Key okey = Key.make("iris.hex");
    Frame fr;
    fr = ParseDataset2.parse(okey, new Key[] { fkey });
    Frame f = DKV.get(okey).get();
    int len = f.numCols();
    Vec[] vv = f.vecs();
    double[] arrayofMeans = new double[len];
    for (int i = 0; i < len; i++) arrayofMeans[i] = vv[i].mean();
    // map reduce call
    FillNasWithMean lr1 = new FillNasWithMean(arrayofMeans).doAll(len, f);
    Key fk = Key.make(f._key.toString() + "_nas_replaced_with_mean");
    Futures fs = new Futures();
    //new frame
    Frame outputFrame = lr1.outputFrame(fk, f.names(), f.domains(), fs);
    fs.blockForPending();
    //puts the new frame in the KV store
    DKV.put(fk, outputFrame, fs);
    fs.blockForPending();
    Log.info(" new output frame        : " + outputFrame);
    //logThisH2OInstanceWebBrowserAddress();
    //sleepForever();
    Frame.delete(okey);
    outputFrame.delete();
}

Example 4 with Vec

use of water.fvec.Vec in project h2o-2 by h2oai.

the class VecDemo method Vec.

@Test
public void Vec() {
    String fileName = "./cookbookData/iris_withNA.csv";
    File file = new File(fileName);
    Key fkey = NFSFileVec.make(file);
    Key okey = Key.make("iris.hex");
    Frame fr;
    fr = ParseDataset2.parse(okey, new Key[] { fkey });
    //accessing the first vector from the frame
    Vec vv = fr.vec(0);
    int loop_indx = 0;
    if (vv.length() > 4) {
        loop_indx = 3;
    }
    //READING AN ELEMENT FROM A VEC
    for (int i = 0; i < loop_indx; i++) {
        long k = i;
        // element at index k returns a double
        double elemnt_D = vv.at(k);
        System.out.println("element at index " + k + " as double: " + elemnt_D);
        if (!Double.isNaN(elemnt_D)) {
            // element at index k returns an (rounded) int, throws if a value is missing
            long elemnt_L = vv.at8(k);
            System.out.println("element at index " + k + " as integer: " + elemnt_L);
        }
    }
    //TESTING WHETHER A VEC OF INTEGERS IS AN ENUM (AKA CATEGORICAL) OR NOT
    for (int i = 0; i < fr.numCols(); i++) {
        Vec vvec = fr.vec(i);
        //check if int
        boolean b = vvec.isInt();
        System.out.println("Is " + i + " an integer column ?" + "  " + b);
        if (b == true) {
            // check if enum
            int cardinality = vvec.cardinality();
            if (cardinality != -1) {
                System.out.println("The vector " + i + " is an enum with cardinality " + cardinality + " and domain names: ");
                //PRINTING THE LIST OF DOMAINS OF AN ENUM VEC (AKA LEVELS OF A CATEGORICAL VEC)
                for (int j = 0; j < cardinality; j++) System.out.println(vvec.domain(j));
            }
        }
    }
    /* This sets the value in a very slow way, because it takes the vector goes to the chunk that has 
		 * the row index, decompress it, updates the value and then compress it again
		*/
    for (int i = 0; i < loop_indx; i++) {
        long k = i;
        double d = 1.23;
        // set element as double
        vv.set(k, d);
        System.out.println("setting element at index " + k + " as double: " + vv.at(k));
        float f = 1.23f;
        // set element as float
        vv.set(k, f);
        System.out.println("setting element at index " + k + " as float: " + vv.at(k));
        long l = 12345678910L;
        // set element as long
        vv.set(k, l);
        System.out.println("setting element at index " + k + " as long: " + vv.at(k));
        // set element as na
        vv.setNA(k);
        System.out.println("setting element at index " + k + " as NAN: " + vv.at(k));
    }
    //UPDATING A VEC ELEMENT WITH AN ENUM VALUE THAT HAS NEVER BEEN USED BEFORE
    Vec vvenum = fr.vec(4);
    final String[] newDomain = new String[] { "x", "y", "z" };
    vvenum.changeDomain(newDomain);
    System.out.println("The changed domain names are: ");
    for (int i = 0; i < vvenum.cardinality(); i++) System.out.println(vvenum.domain(i));
    //fr.vec(4).changeDomain(newDomain);
    //ACCESSING VEC STATS THAT ARE COMPUTED AUTOMATICALLY (LIKE MIN, MAX)
    System.out.println("Min for vector 0: " + vv.min());
    System.out.println("Max for vector 0: " + vv.max());
    System.out.println("Mean for vector 0: " + vv.mean());
    System.out.println("Standard deviation for vector 0: " + vv.sigma());
    System.out.println("NA count for vector 0: " + vv.naCnt());
    //logThisH2OInstanceWebBrowserAddress();
    //sleepForever();	
    //CLEANING THE KV STORE OF ALL DATA
    Frame.delete(okey);
}

Example 5 with Vec

use of water.fvec.Vec in project h2o-2 by h2oai.

the class MnistCanvas method paint.

@Override
public void paint(Graphics g) {
    Layer[] ls = _trainer.layers();
    Vec[] vecs = ((VecsInput) ls[0]).vecs;
    //    Vec resp = ((VecSoftmax) ls[ls.length - 1]).vec;
    int edge = 56, pad = 10;
    int rand = _rand.nextInt((int) vecs[0].length());
    // Side
    {
        BufferedImage in = new BufferedImage(EDGE, EDGE, BufferedImage.TYPE_INT_RGB);
        WritableRaster r = in.getRaster();
        // Input
        int[] pix = new int[PIXELS];
        for (int i = 0; i < pix.length; i++) pix[i] = (int) (vecs[i].at8(rand));
        r.setDataElements(0, 0, EDGE, EDGE, pix);
        g.drawImage(in, pad, pad, null);
    // Labels
    //      g.drawString("" + resp.at8(rand), 10, 50);
    //      g.drawString("RBM " + _level, 10, 70);
    }
    // Outputs
    int offset = pad;
    //    float[] visible = new float[MnistNeuralNetTest.PIXELS];
    //    System.arraycopy(_images, rand * MnistNeuralNetTest.PIXELS, visible, 0, MnistNeuralNetTest.PIXELS);
    //    for( int i = 0; i <= _level; i++ ) {
    //      for( int pass = 0; pass < 10; pass++ ) {
    //        if( i == _level ) {
    //          int[] output = new int[visible.length];
    //          for( int v = 0; v < visible.length; v++ )
    //            output[v] = (int) Math.min(visible[v] * 255, 255);
    //          BufferedImage out = new BufferedImage(MnistNeuralNetTest.EDGE, MnistNeuralNetTest.EDGE,
    //              BufferedImage.TYPE_INT_RGB);
    //          WritableRaster r = out.getRaster();
    //          r.setDataElements(0, 0, MnistNeuralNetTest.EDGE, MnistNeuralNetTest.EDGE, output);
    //          BufferedImage image = new BufferedImage(edge, edge, BufferedImage.TYPE_INT_RGB);
    //          Graphics2D ig = image.createGraphics();
    //          ig.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
    //          ig.clearRect(0, 0, edge, edge);
    //          ig.drawImage(out, 0, 0, edge, edge, null);
    //          ig.dispose();
    //          g.drawImage(image, pad * 2 + MnistNeuralNetTest.EDGE, offset, null);
    //          offset += pad + edge;
    //        }
    //        if( _ls[i]._v != null ) {
    //          float[] hidden = new float[_ls[i]._b.length];
    //          _ls[i].forward(visible, hidden);
    //          visible = _ls[i].generate(hidden);
    //        }
    //      }
    //      float[] t = new float[_ls[i]._b.length];
    //      _ls[i].forward(visible, t);
    //      visible = t;
    //    }
    // Weights
    int buf = EDGE + pad + pad;
    Layer layer = ls[_level];
    double mean = 0;
    int n = layer._w.length;
    for (int i = 0; i < n; i++) mean += layer._w[i];
    mean /= layer._w.length;
    double sigma = 0;
    for (int i = 0; i < layer._w.length; i++) {
        double d = layer._w[i] - mean;
        sigma += d * d;
    }
    sigma = Math.sqrt(sigma / (layer._w.length - 1));
    for (int o = 0; o < layer._b.length; o++) {
        if (o % 10 == 0) {
            offset = pad;
            buf += pad + edge;
        }
        int[] start = new int[layer._previous._a.length];
        for (int i = 0; i < layer._previous._a.length; i++) {
            double w = layer._w[o * layer._previous._a.length + i];
            w = ((w - mean) / sigma) * 200;
            if (w >= 0)
                //GREEN
                start[i] = ((int) Math.min(+w, 255)) << 8;
            else
                //RED
                start[i] = ((int) Math.min(-w, 255)) << 16;
        }
        BufferedImage out = new BufferedImage(EDGE, EDGE, BufferedImage.TYPE_INT_RGB);
        WritableRaster r = out.getRaster();
        r.setDataElements(0, 0, EDGE, EDGE, start);
        BufferedImage resized = new BufferedImage(edge, edge, BufferedImage.TYPE_INT_RGB);
        Graphics2D g2 = resized.createGraphics();
        try {
            g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
            g2.clearRect(0, 0, edge, edge);
            g2.drawImage(out, 0, 0, edge, edge, null);
        } finally {
            g2.dispose();
        }
        g.drawImage(resized, buf, offset, null);
        offset += pad + edge;
    }
}