Examples with NFSFileVec water.fvec.NFSFileVec used on opensource projects

Example 16 with NFSFileVec

use of water.fvec.NFSFileVec in project h2o-3 by h2oai.

the class DeepLearningTest method testMultinomialMNIST.

@Ignore
@Test
public void testMultinomialMNIST() {
    Frame train = null;
    Frame preds = null;
    Frame small = null, large = null;
    DeepLearningModel model = null;
    Scope.enter();
    try {
        File file = FileUtils.locateFile("bigdata/laptop/mnist/train.csv.gz");
        if (file != null) {
            NFSFileVec trainfv = NFSFileVec.make(file);
            train = ParseDataset.parse(Key.make(), trainfv._key);
            int ci = train.find("C785");
            Scope.track(train.replace(ci, train.vecs()[ci].toCategoricalVec()));
            DKV.put(train);
            DeepLearningParameters p = new DeepLearningParameters();
            p._train = train._key;
            // last column is the response
            p._response_column = "C785";
            p._activation = DeepLearningParameters.Activation.RectifierWithDropout;
            p._hidden = new int[] { 50, 50 };
            p._epochs = 1;
            p._adaptive_rate = false;
            p._rate = 0.005;
            p._sparse = true;
            model = new DeepLearning(p).trainModel().get();
            FrameSplitter fs = new FrameSplitter(train, new double[] { 0.0001 }, new Key[] { Key.make("small"), Key.make("large") }, null);
            fs.compute2();
            small = fs.getResult()[0];
            large = fs.getResult()[1];
            preds = model.score(small);
            //remove label, keep only probs
            preds.remove(0);
            //actual
            Vec labels = small.vec("C785");
            //actual
            String[] fullDomain = train.vec("C785").domain();
            ModelMetricsMultinomial mm = ModelMetricsMultinomial.make(preds, labels, fullDomain);
            Log.info(mm.toString());
        }
    } catch (Throwable t) {
        t.printStackTrace();
        throw t;
    } finally {
        if (model != null)
            model.delete();
        if (preds != null)
            preds.remove();
        if (train != null)
            train.remove();
        if (small != null)
            small.delete();
        if (large != null)
            large.delete();
        Scope.exit();
    }
}

Example 17 with NFSFileVec

use of water.fvec.NFSFileVec in project h2o-3 by h2oai.

the class KVTest method testMultiMbFile.

// ---
// Map in h2o.jar - a multi-megabyte file - into a NFSFileVec
// Run a distributed byte histogram.
@Test
public void testMultiMbFile() {
    long start = System.currentTimeMillis();
    NFSFileVec nfs = null;
    try {
        File file = FileUtils.locateFile("build/h2o-core.jar");
        // Nothing to test
        if (file == null)
            return;
        // Return a Key mapping to a NFSFileVec over the file
        nfs = NFSFileVec.make(file);
        ByteHisto bh = new ByteHisto().doAll(nfs);
        int sum = water.util.ArrayUtils.sum(bh._x);
        assertEquals(file.length(), sum);
    } finally {
        // remove from DKV
        if (nfs != null)
            nfs.remove();
    }
    System.out.println("MultiMbFile " + (System.currentTimeMillis() - start));
}

Example 18 with NFSFileVec

use of water.fvec.NFSFileVec in project h2o-3 by h2oai.

the class ParseFolderTest method testSameFile.

@Test
public void testSameFile() {
    File f = FileUtils.locateFile("smalldata/iris/iris_wheader.csv");
    NFSFileVec nfs1 = NFSFileVec.make(f);
    NFSFileVec nfs2 = NFSFileVec.make(f);
    Frame fr = null;
    try {
        fr = ParseDataset.parse(Key.make(), new Key[] { nfs1._key, nfs2._key }, false, /*delete on done*/
        false, ParseSetup.GUESS_HEADER);
    } finally {
        if (fr != null)
            fr.delete();
        if (nfs1 != null)
            nfs1.remove();
    }
}

Example 19 with NFSFileVec

use of water.fvec.NFSFileVec in project h2o-3 by h2oai.

the class ParseCompressedAndXLSTest method testXLSBadArgs.

@Test
public void testXLSBadArgs() {
    Frame k1 = null;
    try {
        NFSFileVec nfs = TestUtil.makeNfsFileVec("smalldata/airlines/AirlinesTest.csv.zip");
        byte[] ctypes = new byte[12];
        for (int i = 0; i < 12; i++) ctypes[i] = Vec.T_NUM;
        ParseSetup setup = new ParseSetup(XLS_INFO, // sep; ascii '4'
        (byte) 52, // singleQuotes
        true, // check header
        ParseSetup.NO_HEADER, // ncols
        12, new String[] { "fYear", "fMonth", "fDayofMonth", "fDayOfWeek", "DepTime", "ArrTime", "UniqueCarrier", "Origin", "Dest", "Distance", "IsDepDelayed", "IsDepDelayed_REC" }, ctypes, null, null, null);
        try {
            k1 = ParseDataset.parse(Key.make(), new Key[] { nfs._key }, true, setup, true)._job.get();
            // fail - should've thrown
            assertTrue("Should have thrown ParseException since file isn't XLS file", false);
            k1.delete();
        } catch (Throwable t) {
            assertTrue(t instanceof ParseDataset.H2OParseException || t.getCause() instanceof ParseDataset.H2OParseException);
        }
    } finally {
        if (k1 != null)
            k1.delete();
    }
}

Example 20 with NFSFileVec

use of water.fvec.NFSFileVec in project h2o-2 by h2oai.

the class MRThrow method testContinuationThrow.

@Test
public void testContinuationThrow() throws InterruptedException, ExecutionException {
    File file = find_test_file("target/h2o.jar");
    Key h2okey = load_test_file(file);
    NFSFileVec nfs = DKV.get(h2okey).get();
    try {
        for (int i = 0; i < H2O.CLOUD._memary.length; ++i) {
            ByteHistoThrow bh = new ByteHistoThrow();
            bh._throwAt = H2O.CLOUD._memary[i].toString();
            final boolean[] ok = new boolean[] { false };
            try {
                bh.setCompleter(new CountedCompleter() {

                    @Override
                    public void compute() {
                    }

                    @Override
                    public boolean onExceptionalCompletion(Throwable ex, CountedCompleter cc) {
                        ok[0] = ex.getMessage().contains("test");
                        return true;
                    }
                });
                // invoke should throw DistrDTibutedException wrapped up in RunTimeException
                bh.dfork(nfs).get();
                assertTrue(ok[0]);
            } catch (ExecutionException eex) {
                assertTrue(eex.getCause().getMessage().contains("test"));
            } catch (Throwable ex) {
                ex.printStackTrace();
                fail("Unexpected exception" + ex.toString());
            }
        }
    } finally {
        // so once a map() call fails, other map calls can lazily load data after we call delete()
        try {
            Thread.sleep(100);
        } catch (InterruptedException ignore) {
        }
        Lockable.delete(h2okey);
    }
}