Examples with FileVec water.fvec.FileVec used on opensource projects

Example 1 with FileVec

use of water.fvec.FileVec in project h2o-3 by h2oai.

the class PersistS3Test method testS3Import.

@Test
public void testS3Import() throws Exception {
    Scope.enter();
    try {
        Key k = H2O.getPM().anyURIToKey(new URI("s3://h2o-public-test-data/smalldata/airlines/AirlinesTrain.csv.zip"));
        Frame fr = DKV.getGet(k);
        FileVec v = (FileVec) fr.anyVec();
        // make sure we have some chunks
        int chunkSize = (int) (v.length() / 3);
        v.setChunkSize(fr, chunkSize);
        long xor = new XORTask().doAll(v)._res;
        Key k2 = H2O.getPM().anyURIToKey(new URI(FileUtils.getFile("smalldata/airlines/AirlinesTrain.csv.zip").getAbsolutePath()));
        FileVec v2 = DKV.getGet(k2);
        assertEquals(v2.length(), v.length());
        assertVecEquals(v, v2, 0);
        // make sure we have some chunks
        v2.setChunkSize(chunkSize);
        long xor2 = new XORTask().doAll(v2)._res;
        assertEquals(xor2, xor);
        fr.delete();
        v2.remove();
    } finally {
        Scope.exit();
    }
}

Example 2 with FileVec

use of water.fvec.FileVec in project h2o-3 by h2oai.

the class ZipUtil method getDecompressionRatio.

/**
   *   When a file is a zip file that contains multiple files, this method will return the decompression ratio.
   *
   * @param bv
   * @return
   */
static float getDecompressionRatio(ByteVec bv) {
    long totalSize = 0L;
    long totalCompSize = 0L;
    if (bv instanceof FileVec) {
        String strPath = getPathForKey(((FileVec) bv)._key);
        try {
            ZipFile zipFile = new ZipFile(strPath);
            Enumeration<? extends ZipEntry> entries = zipFile.entries();
            while (entries.hasMoreElements()) {
                ZipEntry entry = entries.nextElement();
                if (!entry.isDirectory()) {
                    // add file to list to parse if not a directory.
                    totalSize = totalSize + entry.getSize();
                    totalCompSize = totalCompSize + entry.getCompressedSize();
                }
            }
            zipFile.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    if (// something is wrong.  Return no compression.
    totalCompSize == 0)
        return 1;
    else
        return totalSize / totalCompSize;
}

Example 3 with FileVec

use of water.fvec.FileVec in project h2o-3 by h2oai.

the class ZipUtil method getFileNames.

static ArrayList<String> getFileNames(ByteVec bv) {
    ArrayList<String> fileList = new ArrayList<String>();
    if (bv instanceof FileVec) {
        String strPath = getPathForKey(((FileVec) bv)._key);
        try {
            ZipFile zipFile = new ZipFile(strPath);
            Enumeration<? extends ZipEntry> entries = zipFile.entries();
            while (entries.hasMoreElements()) {
                ZipEntry entry = entries.nextElement();
                if (!entry.isDirectory()) {
                    // add file to list to parse if not a directory.
                    fileList.add(entry.getName());
                }
            }
            zipFile.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    return fileList;
}