Examples with Key water.Key used on opensource projects

Example 21 with Key

use of water.Key in project h2o-3 by h2oai.

the class C1ChunkTest method test_setNA.

@Test
public void test_setNA() {
    // Create a vec with one chunk, and set its numbers
    Key key = Vec.newKey();
    Vec vec = new Vec(key, Vec.ESPC.rowLayout(key, new long[] { 0, 15 })).makeZero();
    int[] vals = new int[] { 0, 1, 0, 5, 0, 0, 0, 21, 0, 111, 0, 8, 0, 1 };
    Vec.Writer w = vec.open();
    for (int i = 0; i < vals.length; ++i) w.set(i, vals[i]);
    //extra NA to make this a C1Chunk, not a C1NChunk
    w.setNA(14);
    w.close();
    Chunk cc = vec.chunkForChunkIdx(0);
    assert cc instanceof C1Chunk;
    Futures fs = new Futures();
    fs.blockForPending();
    for (int i = 0; i < vals.length; ++i) Assert.assertEquals(vals[i], cc.at8(i));
    for (int i = 0; i < vals.length; ++i) Assert.assertEquals(vals[i], cc.at8_abs(i));
    int[] NAs = new int[] { 1, 5, 2, 14 };
    int[] notNAs = new int[] { 0, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13 };
    for (int na : NAs) cc.setNA(na);
    for (int na : NAs) cc.setNA_abs(na);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    NewChunk nc = new NewChunk(null, 0);
    cc.extractRows(nc, 0, (int) vec.length());
    Assert.assertEquals(vals.length + 1, nc._sparseLen);
    Assert.assertEquals(vals.length + 1, nc._len);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    Chunk cc2 = nc.compress();
    Assert.assertEquals(vals.length + 1, cc._len);
    Assert.assertTrue(cc2 instanceof C1Chunk);
    for (int na : NAs) Assert.assertTrue(cc.isNA(na));
    for (int na : NAs) Assert.assertTrue(cc.isNA_abs(na));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA(notna));
    for (int notna : notNAs) Assert.assertTrue(!cc.isNA_abs(notna));
    Assert.assertTrue(Arrays.equals(cc._mem, cc2._mem));
    vec.remove();
}

Example 22 with Key

use of water.Key in project h2o-3 by h2oai.

the class ParseFolderTestBig method testBIGSVM.

@Test
@Ignore
public void testBIGSVM() {
    String fname = "bigdata/cust_K/1m.svm";
    Frame k1 = null;
    try {
        File f = FileUtils.getFile(fname);
        NFSFileVec nfs = NFSFileVec.make(f);
        Job<Frame> job = ParseDataset.parse(Key.make("BIGSVM.hex"), new Key[] { nfs._key }, true, ParseSetup.guessSetup(new Key[] { nfs._key }, false, ParseSetup.GUESS_HEADER), false)._job;
        while (job.progress() < 1.0) {
            System.out.print(((int) (job.progress() * 1000.0)) / 10.0 + "% ");
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ignore) {
            /*comment to disable ideaJ warning*/
            }
        }
        System.out.println();
        k1 = job.get();
        System.out.println(k1.toString());
    } catch (IOException ioe) {
        Assert.fail("File not found: " + fname + " - " + ioe.getMessage());
    } finally {
        if (k1 != null)
            k1.delete();
    }
}

Example 23 with Key

use of water.Key in project h2o-3 by h2oai.

the class ParseProgressTest method testCovtype.

// Attempt a multi-jvm parse of covtype.
// Silently exits if it cannot find covtype.
@Test
public void testCovtype() {
    String[] covtype_locations = new String[] { "../datasets/UCI/UCI-large/covtype/covtype.data", "../../datasets/UCI/UCI-large/covtype/covtype.data", "../datasets/UCI/UCI-large/covtype/covtype.data.gz", "../demo/UCI-large/covtype/covtype.data" };
    File f = null;
    for (String covtype_location : covtype_locations) {
        f = FileUtils.locateFile(covtype_location);
        if (f != null && f.exists())
            break;
    }
    if (f == null || !f.exists()) {
        System.out.println("Could not find covtype.data, skipping ParseProgressTest.testCovtype()");
        return;
    }
    FileIntegrityChecker c = FileIntegrityChecker.check(f);
    // Exactly 1 file
    Assert.assertEquals(1, c.size());
    Key k = c.syncDirectory(null, null, null, null);
    Assert.assertEquals(true, k != null);
    Frame fr = ParseDataset.parse(Key.make(), k);
    Assert.assertEquals(55, fr.numCols());
    Assert.assertEquals(581012, fr.numRows());
    fr.delete();
}

Example 24 with Key

use of water.Key in project h2o-3 by h2oai.

the class ParseTimeTest method testDayParseNoTime1.

@Test
public void testDayParseNoTime1() {
    DateTimeZone pst = DateTimeZone.forID("America/Los_Angeles");
    DateTimeZone localTZ = DateTimeZone.getDefault();
    // Just yyyy-mm-dd, no time
    String data = "Date\n" + "2014-1-23\n" + "2014-1-24\n" + "2014-1-23\n" + "2014-1-24\n";
    Key k1 = ParserTest.makeByteVec(data);
    Key r1 = Key.make("r1");
    Frame fr = ParseDataset.parse(r1, k1);
    Assert.assertTrue(fr.vec(0).get_type_str().equals("Time"));
    long[] exp = new long[] { // Date, note: these ms counts all presume PST
    1390464000000L, 1390550400000L, 1390464000000L, 1390550400000L };
    for (// Adjust exp[] to local time
    int i = 0; // Adjust exp[] to local time
    i < exp.length; // Adjust exp[] to local time
    i++) exp[i] += pst.getOffset(exp[i]) - localTZ.getOffset(exp[i]);
    Vec vec = fr.vec("Date");
    for (int i = 0; i < exp.length; i++) Assert.assertEquals(exp[i], vec.at8(i));
    fr.delete();
}

Example 25 with Key

use of water.Key in project h2o-3 by h2oai.

the class ParseTimeTest method testTimeParse3.

@Test
public void testTimeParse3() {
    DateTimeZone pst = DateTimeZone.forID("America/Los_Angeles");
    DateTimeZone localTZ = DateTimeZone.getDefault();
    String[] data = new String[] { "12Jun10:10:00:00", "12JUN2010:10:00:00", // Embedded blank, no  quotes
    "12JUN2010 10:00:00", // Embedded blank, yes quotes
    "\"12JUN2010 10:00:00\"", // Embedded blank, no  quotes
    "12JUN2010:10:00:00 PM", // Embedded blank, yes quotes
    "\"12JUN2010:10:00:00 PM\"", "12JUN2010:10:00:00.123456789", "\"12JUN2010:10:00:00.123456789 PM\"", "12June2010", // Embedded blank, no  quotes
    "24-MAR-14 06.10.48.000000000 PM", // Embedded blank, yes quotes
    "\"24-MAR-14 06.10.48.000000000 PM\"", "\"24-MAR-14 06.10.48.000000000PM\"", "\"24-MAR-14:06.10.48.123 AM\"", "24-MAR-14:06.10.48.123AM", "24-MAR-14:06.10.48.000000000", "\"24-MAR-14:06.10:48.000 PM\"", "\"24MAR14:06.10:48.000 PM\"", // should handle days with one digit
    "\"4MAR2014:06.10:48.000 PM\"", // should assume 1978
    "\"24MAR78:06.10:48.000 PM\"", // should be a negative time, pre-Epoch
    "\"24MAR1968:06.10:48.000 PM\"", // Evil trailing blank
    "2015-12-03 15:43:21.654321 ", // Evil trailing blank, quoted
    "\"2015-12-03 15:43:21.654321 \"", // No dash '-' separator between yyyyMMdd, and then one between dd-HH
    "20151203-15:43:21.654" };
    double[][] exp = new double[][] { // These ms counts all presume PST
    d(1276362000000L), d(1276362000000L), d(1276362000000L), d(1276362000000L), d(1276405200000L), d(1276405200000L), d(1276362000123L), d(1276405200123L), d(1276326000000L), d(1395709848000L), d(1395709848000L), d(1395709848000L), d(1395666648123L), d(1395666648123L), d(1395666648000L), d(1395709848000L), d(1395709848000L), d(1393985448000L), d(259639848000L), d(-55892952000L), d(1449186201654L), d(1449186201654L), d(1449186201654L) };
    StringBuilder sb1 = new StringBuilder();
    for (String ds : data) sb1.append(ds).append("\n");
    Key[] k1 = new Key[] { ParserTest.makeByteVec(sb1.toString()) };
    Key r1 = Key.make("r1");
    ParseSetup ps = ParseSetup.guessSetup(k1, false, 0);
    ps._separator = ',';
    ps._number_columns = 1;
    Frame dataFrame = ParseDataset.parse(r1, k1, true, ps);
    for (// Adjust exp[][] to local time
    int i = 0; // Adjust exp[][] to local time
    i < exp.length; // Adjust exp[][] to local time
    i++) for (int j = 0; j < 1; j++) exp[i][j] += pst.getOffset((long) exp[i][j]) - localTZ.getOffset((long) exp[i][j]);
    //File items will be converted to ms for local timezone
    ParserTest.testParsed(dataFrame, exp, exp.length);
}