Search in sources :

Example 1 with ZipFile

use of java.util.zip.ZipFile in project hadoop by apache.

the class FileUtil method unZip.

/**
   * Given a File input it will unzip the file in a the unzip directory
   * passed as the second parameter
   * @param inFile The zip file as input
   * @param unzipDir The unzip directory where to unzip the zip file.
   * @throws IOException
   */
public static void unZip(File inFile, File unzipDir) throws IOException {
    Enumeration<? extends ZipEntry> entries;
    ZipFile zipFile = new ZipFile(inFile);
    try {
        entries = zipFile.entries();
        while (entries.hasMoreElements()) {
            ZipEntry entry = entries.nextElement();
            if (!entry.isDirectory()) {
                InputStream in = zipFile.getInputStream(entry);
                try {
                    File file = new File(unzipDir, entry.getName());
                    if (!file.getParentFile().mkdirs()) {
                        if (!file.getParentFile().isDirectory()) {
                            throw new IOException("Mkdirs failed to create " + file.getParentFile().toString());
                        }
                    }
                    OutputStream out = new FileOutputStream(file);
                    try {
                        byte[] buffer = new byte[8192];
                        int i;
                        while ((i = in.read(buffer)) != -1) {
                            out.write(buffer, 0, i);
                        }
                    } finally {
                        out.close();
                    }
                } finally {
                    in.close();
                }
            }
        }
    } finally {
        zipFile.close();
    }
}
Also used : ZipFile(java.util.zip.ZipFile) GZIPInputStream(java.util.zip.GZIPInputStream) BufferedInputStream(java.io.BufferedInputStream) TarArchiveInputStream(org.apache.commons.compress.archivers.tar.TarArchiveInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) ZipEntry(java.util.zip.ZipEntry) BufferedOutputStream(java.io.BufferedOutputStream) JarOutputStream(java.util.jar.JarOutputStream) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) ZipFile(java.util.zip.ZipFile) File(java.io.File)

Example 2 with ZipFile

use of java.util.zip.ZipFile in project cogtool by cogtool.

the class ObjectPersister method load.

// registerForPersistence
/**
     * Load an Object from a file.  This method assumes that all error checking
     * and prompting/correction for the validity of the file has already
     * been completed.
     *
     * @param src the File location of the saved Object
     * @return a new instantiated Object
     * @throws java.io.IOException if any file operation fails or the SAX XML
     *         parser fails
     */
public Object load(File src) throws IOException {
    // See if an object is already loaded for the given file
    String canonicalFileName = src.getCanonicalPath();
    PersistInfo info = getInfoByName(canonicalFileName);
    if (info != null) {
        return info.obj;
    }
    // Attempt to create a file of 3x size in the temp directory
    // to hold the expanded XML serialization.
    File sizeCheckFile = File.createTempFile(tmpFilePrefix, ".size", tmpDir);
    try {
        checkDiskSpace(sizeCheckFile, 3 * src.length(), "Not enough temp space on disk to open file: " + src);
    } finally {
        sizeCheckFile.delete();
    }
    // If we're here, no exception was thrown; create checkpoint directory
    File chkptFile = createTempDirectory();
    // Open file as ZipFile and decompress
    ZipFile zip = null;
    try {
        zip = new ZipFile(src);
        // Unzip the file to the checkpoint directory
        ZipUtil.unzip(zip, chkptFile);
    } catch (ZipException ex) {
        IOException newE = new IOException("load encountered zip compression error");
        newE.initCause(ex);
        throw newE;
    } finally {
        if (zip != null) {
            zip.close();
        }
    }
    // Load object from the expanded XML serialization
    ObjectLoader l = new ObjectLoader();
    Object obj = null;
    Reader reader = null;
    try {
        reader = new InputStreamReader(new FileInputStream(new File(chkptFile, PERSIST_FILE)), "UTF-8");
        Collection<?> objSet = l.load(new InputSource(reader), null);
        // There should be only one top-level object
        Iterator<?> objs = objSet.iterator();
        if (objs.hasNext()) {
            obj = objs.next();
        }
        // Register this file for future lookup, both by object
        // and by file name
        info = new PersistInfo(obj, chkptFile, src);
        objInfos.put(obj, info);
        fileInfos.put(canonicalFileName, info);
    } catch (ParserConfigurationException e) {
        IOException newE = new IOException("load encountered parser error");
        newE.initCause(e);
        throw newE;
    } catch (SAXException e) {
        IOException newE = new IOException("load encountered SAX error");
        newE.initCause(e);
        throw newE;
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
    return obj;
}
Also used : InputSource(org.xml.sax.InputSource) InputStreamReader(java.io.InputStreamReader) Reader(java.io.Reader) InputStreamReader(java.io.InputStreamReader) FileReader(java.io.FileReader) ZipException(java.util.zip.ZipException) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) SAXException(org.xml.sax.SAXException) ZipFile(java.util.zip.ZipFile) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) ZipFile(java.util.zip.ZipFile)

Example 3 with ZipFile

use of java.util.zip.ZipFile in project che by eclipse.

the class ZipUtilsTest method testGetResources.

@Test
public void testGetResources() throws Exception {
    URL testJar = ZipUtilsTest.class.getResource("/che/che.jar");
    @SuppressWarnings("unchecked") Consumer<InputStream> consumer = mock(Consumer.class);
    ZipUtils.getResources(new ZipFile(testJar.getFile()), Pattern.compile(".*[//]?codenvy/[^//]+[.]json"), consumer);
    verify(consumer, times(2)).accept(any(InputStream.class));
}
Also used : ZipFile(java.util.zip.ZipFile) InputStream(java.io.InputStream) URL(java.net.URL) Test(org.testng.annotations.Test)

Example 4 with ZipFile

use of java.util.zip.ZipFile in project druid by druid-io.

the class CompressionUtils method unzip.

/**
   * Unzip the pulled file to an output directory. This is only expected to work on zips with lone files, and is not intended for zips with directory structures.
   *
   * @param pulledFile The file to unzip
   * @param outDir     The directory to store the contents of the file.
   *
   * @return a FileCopyResult of the files which were written to disk
   *
   * @throws IOException
   */
public static FileUtils.FileCopyResult unzip(final File pulledFile, final File outDir) throws IOException {
    if (!(outDir.exists() && outDir.isDirectory())) {
        throw new ISE("outDir[%s] must exist and be a directory", outDir);
    }
    log.info("Unzipping file[%s] to [%s]", pulledFile, outDir);
    final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult();
    try (final ZipFile zipFile = new ZipFile(pulledFile)) {
        final Enumeration<? extends ZipEntry> enumeration = zipFile.entries();
        while (enumeration.hasMoreElements()) {
            final ZipEntry entry = enumeration.nextElement();
            result.addFiles(FileUtils.retryCopy(new ByteSource() {

                @Override
                public InputStream openStream() throws IOException {
                    return new BufferedInputStream(zipFile.getInputStream(entry));
                }
            }, new File(outDir, entry.getName()), FileUtils.IS_EXCEPTION, DEFAULT_RETRY_COUNT).getFiles());
        }
    }
    return result;
}
Also used : GZIPInputStream(java.util.zip.GZIPInputStream) BufferedInputStream(java.io.BufferedInputStream) ZipInputStream(java.util.zip.ZipInputStream) FilterInputStream(java.io.FilterInputStream) InputStream(java.io.InputStream) ZipEntry(java.util.zip.ZipEntry) IOException(java.io.IOException) ZipFile(java.util.zip.ZipFile) BufferedInputStream(java.io.BufferedInputStream) ByteSource(com.google.common.io.ByteSource) File(java.io.File) ZipFile(java.util.zip.ZipFile)

Example 5 with ZipFile

use of java.util.zip.ZipFile in project deeplearning4j by deeplearning4j.

the class WordVectorSerializer method writeWord2VecModel.

/**
     * This method saves Word2Vec model into compressed zip file and sends it to output stream
     * PLEASE NOTE: This method saves FULL model, including syn0 AND syn1
     *
     */
public static void writeWord2VecModel(Word2Vec vectors, OutputStream stream) throws IOException {
    ZipOutputStream zipfile = new ZipOutputStream(new BufferedOutputStream(new CloseShieldOutputStream(stream)));
    ZipEntry syn0 = new ZipEntry("syn0.txt");
    zipfile.putNextEntry(syn0);
    // writing out syn0
    File tempFileSyn0 = File.createTempFile("word2vec", "0");
    tempFileSyn0.deleteOnExit();
    writeWordVectors(vectors.lookupTable(), tempFileSyn0);
    BufferedInputStream fis = new BufferedInputStream(new FileInputStream(tempFileSyn0));
    writeEntry(fis, zipfile);
    fis.close();
    // writing out syn1
    File tempFileSyn1 = File.createTempFile("word2vec", "1");
    tempFileSyn1.deleteOnExit();
    INDArray syn1 = ((InMemoryLookupTable<VocabWord>) vectors.getLookupTable()).getSyn1();
    if (syn1 != null)
        try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileSyn1))) {
            for (int x = 0; x < syn1.rows(); x++) {
                INDArray row = syn1.getRow(x);
                StringBuilder builder = new StringBuilder();
                for (int i = 0; i < row.length(); i++) {
                    builder.append(row.getDouble(i)).append(" ");
                }
                writer.println(builder.toString().trim());
            }
        }
    ZipEntry zSyn1 = new ZipEntry("syn1.txt");
    zipfile.putNextEntry(zSyn1);
    fis = new BufferedInputStream(new FileInputStream(tempFileSyn1));
    writeEntry(fis, zipfile);
    fis.close();
    // writing out syn1
    File tempFileSyn1Neg = File.createTempFile("word2vec", "n");
    tempFileSyn1Neg.deleteOnExit();
    INDArray syn1Neg = ((InMemoryLookupTable<VocabWord>) vectors.getLookupTable()).getSyn1Neg();
    if (syn1Neg != null)
        try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileSyn1Neg))) {
            for (int x = 0; x < syn1Neg.rows(); x++) {
                INDArray row = syn1Neg.getRow(x);
                StringBuilder builder = new StringBuilder();
                for (int i = 0; i < row.length(); i++) {
                    builder.append(row.getDouble(i)).append(" ");
                }
                writer.println(builder.toString().trim());
            }
        }
    ZipEntry zSyn1Neg = new ZipEntry("syn1Neg.txt");
    zipfile.putNextEntry(zSyn1Neg);
    fis = new BufferedInputStream(new FileInputStream(tempFileSyn1Neg));
    writeEntry(fis, zipfile);
    fis.close();
    File tempFileCodes = File.createTempFile("word2vec", "h");
    tempFileCodes.deleteOnExit();
    ZipEntry hC = new ZipEntry("codes.txt");
    zipfile.putNextEntry(hC);
    // writing out huffman tree
    try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileCodes))) {
        for (int i = 0; i < vectors.getVocab().numWords(); i++) {
            VocabWord word = vectors.getVocab().elementAtIndex(i);
            StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ");
            for (int code : word.getCodes()) {
                builder.append(code).append(" ");
            }
            writer.println(builder.toString().trim());
        }
    }
    fis = new BufferedInputStream(new FileInputStream(tempFileCodes));
    writeEntry(fis, zipfile);
    fis.close();
    File tempFileHuffman = File.createTempFile("word2vec", "h");
    tempFileHuffman.deleteOnExit();
    ZipEntry hP = new ZipEntry("huffman.txt");
    zipfile.putNextEntry(hP);
    // writing out huffman tree
    try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileHuffman))) {
        for (int i = 0; i < vectors.getVocab().numWords(); i++) {
            VocabWord word = vectors.getVocab().elementAtIndex(i);
            StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ");
            for (int point : word.getPoints()) {
                builder.append(point).append(" ");
            }
            writer.println(builder.toString().trim());
        }
    }
    fis = new BufferedInputStream(new FileInputStream(tempFileHuffman));
    writeEntry(fis, zipfile);
    fis.close();
    File tempFileFreqs = File.createTempFile("word2vec", "f");
    tempFileFreqs.deleteOnExit();
    ZipEntry hF = new ZipEntry("frequencies.txt");
    zipfile.putNextEntry(hF);
    // writing out word frequencies
    try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileFreqs))) {
        for (int i = 0; i < vectors.getVocab().numWords(); i++) {
            VocabWord word = vectors.getVocab().elementAtIndex(i);
            StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ").append(word.getElementFrequency()).append(" ").append(vectors.getVocab().docAppearedIn(word.getLabel()));
            writer.println(builder.toString().trim());
        }
    }
    fis = new BufferedInputStream(new FileInputStream(tempFileFreqs));
    writeEntry(fis, zipfile);
    fis.close();
    ZipEntry config = new ZipEntry("config.json");
    zipfile.putNextEntry(config);
    //log.info("Current config: {}", vectors.getConfiguration().toJson());
    writeEntry(new ByteArrayInputStream(vectors.getConfiguration().toJson().getBytes()), zipfile);
    zipfile.flush();
    zipfile.close();
    try {
        tempFileCodes.delete();
        tempFileFreqs.delete();
        tempFileHuffman.delete();
        tempFileSyn0.delete();
        tempFileSyn1.delete();
        tempFileSyn1Neg.delete();
    } catch (Exception e) {
    //
    }
}
Also used : ZipEntry(java.util.zip.ZipEntry) VocabWord(org.deeplearning4j.models.word2vec.VocabWord) DL4JInvalidInputException(org.deeplearning4j.exception.DL4JInvalidInputException) ND4JIllegalStateException(org.nd4j.linalg.exception.ND4JIllegalStateException) InMemoryLookupTable(org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable) INDArray(org.nd4j.linalg.api.ndarray.INDArray) ZipOutputStream(java.util.zip.ZipOutputStream) ZipFile(java.util.zip.ZipFile)

Aggregations

ZipFile (java.util.zip.ZipFile)580 ZipEntry (java.util.zip.ZipEntry)414 File (java.io.File)261 IOException (java.io.IOException)189 InputStream (java.io.InputStream)127 FileOutputStream (java.io.FileOutputStream)98 ZipOutputStream (java.util.zip.ZipOutputStream)89 Test (org.junit.Test)88 FileInputStream (java.io.FileInputStream)57 ArrayList (java.util.ArrayList)44 Enumeration (java.util.Enumeration)42 BufferedInputStream (java.io.BufferedInputStream)38 BufferedOutputStream (java.io.BufferedOutputStream)35 ZipException (java.util.zip.ZipException)32 ClassReader (org.objectweb.asm.ClassReader)29 OutputStream (java.io.OutputStream)27 JarFile (java.util.jar.JarFile)26 ZipInputStream (java.util.zip.ZipInputStream)26 FileNotFoundException (java.io.FileNotFoundException)23 Path (java.nio.file.Path)23