Search in sources :

Example 21 with FileOutputStream

use of java.io.FileOutputStream in project druid by druid-io.

the class FileSmoosher method close.

@Override
public void close() throws IOException {
    //book keeping checks on created file.
    if (!completedFiles.isEmpty() || !filesInProcess.isEmpty()) {
        for (File file : completedFiles) {
            if (!file.delete()) {
                LOG.warn("Unable to delete file [%s]", file);
            }
        }
        for (File file : filesInProcess) {
            if (!file.delete()) {
                LOG.warn("Unable to delete file [%s]", file);
            }
        }
        throw new ISE("[%d] writers in progress and [%d] completed writers needs to be closed before closing smoosher.", filesInProcess.size(), completedFiles.size());
    }
    if (currOut != null) {
        currOut.close();
    }
    File metaFile = metaFile(baseDir);
    try (Writer out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(metaFile), Charsets.UTF_8))) {
        out.write(String.format("v1,%d,%d", maxChunkSize, outFiles.size()));
        out.write("\n");
        for (Map.Entry<String, Metadata> entry : internalFiles.entrySet()) {
            final Metadata metadata = entry.getValue();
            out.write(joiner.join(entry.getKey(), metadata.getFileNum(), metadata.getStartOffset(), metadata.getEndOffset()));
            out.write("\n");
        }
    }
}
Also used : FileOutputStream(java.io.FileOutputStream) ISE(io.druid.java.util.common.ISE) OutputStreamWriter(java.io.OutputStreamWriter) File(java.io.File) Map(java.util.Map) OutputStreamWriter(java.io.OutputStreamWriter) BufferedWriter(java.io.BufferedWriter) Writer(java.io.Writer) BufferedWriter(java.io.BufferedWriter)

Example 22 with FileOutputStream

use of java.io.FileOutputStream in project druid by druid-io.

the class CompressionUtilsTest method testStreamErrorGunzip.

@Test(expected = IOException.class)
public void testStreamErrorGunzip() throws Exception {
    final File tmpDir = temporaryFolder.newFolder("testGoodGzipByteSource");
    final File gzFile = new File(tmpDir, testFile.getName() + ".gz");
    Assert.assertFalse(gzFile.exists());
    CompressionUtils.gzip(Files.asByteSource(testFile), Files.asByteSink(gzFile), Predicates.<Throwable>alwaysTrue());
    Assert.assertTrue(gzFile.exists());
    try (final InputStream inputStream = CompressionUtils.gzipInputStream(new FileInputStream(gzFile))) {
        assertGoodDataStream(inputStream);
    }
    if (testFile.exists() && !testFile.delete()) {
        throw new RuntimeException(String.format("Unable to delete file [%s]", testFile.getAbsolutePath()));
    }
    Assert.assertFalse(testFile.exists());
    final AtomicLong flushes = new AtomicLong(0L);
    CompressionUtils.gunzip(new FileInputStream(gzFile), new FilterOutputStream(new FileOutputStream(testFile) {

        @Override
        public void flush() throws IOException {
            if (flushes.getAndIncrement() > 0) {
                super.flush();
            } else {
                throw new IOException("Test exception");
            }
        }
    }));
}
Also used : AtomicLong(java.util.concurrent.atomic.AtomicLong) GZIPInputStream(java.util.zip.GZIPInputStream) FilterInputStream(java.io.FilterInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) FilterOutputStream(java.io.FilterOutputStream) File(java.io.File) FileInputStream(java.io.FileInputStream) Test(org.junit.Test)

Example 23 with FileOutputStream

use of java.io.FileOutputStream in project druid by druid-io.

the class ByteBufferUtilsTest method testUnmapDoesntCrashJVM.

@Test
public void testUnmapDoesntCrashJVM() throws Exception {
    final File file = temporaryFolder.newFile("some_mmap_file");
    try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
        final byte[] data = new byte[4096];
        Arrays.fill(data, (byte) 0x5A);
        os.write(data);
    }
    final MappedByteBuffer mappedByteBuffer = Files.map(file);
    Assert.assertEquals((byte) 0x5A, mappedByteBuffer.get(0));
    ByteBufferUtils.unmap(mappedByteBuffer);
    ByteBufferUtils.unmap(mappedByteBuffer);
}
Also used : MappedByteBuffer(java.nio.MappedByteBuffer) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) BufferedOutputStream(java.io.BufferedOutputStream) FileOutputStream(java.io.FileOutputStream) File(java.io.File) BufferedOutputStream(java.io.BufferedOutputStream) Test(org.junit.Test)

Example 24 with FileOutputStream

use of java.io.FileOutputStream in project druid by druid-io.

the class CompressionUtilsTest method testGoodZipStream.

@Test
public void testGoodZipStream() throws IOException {
    final File tmpDir = temporaryFolder.newFolder("testGoodZipStream");
    final File zipFile = new File(tmpDir, "compressionUtilTest.zip");
    CompressionUtils.zip(testDir, new FileOutputStream(zipFile));
    final File newDir = new File(tmpDir, "newDir");
    newDir.mkdir();
    CompressionUtils.unzip(new FileInputStream(zipFile), newDir);
    final Path newPath = Paths.get(newDir.getAbsolutePath(), testFile.getName());
    Assert.assertTrue(newPath.toFile().exists());
    try (final FileInputStream inputStream = new FileInputStream(newPath.toFile())) {
        assertGoodDataStream(inputStream);
    }
}
Also used : Path(java.nio.file.Path) FileOutputStream(java.io.FileOutputStream) File(java.io.File) FileInputStream(java.io.FileInputStream) Test(org.junit.Test)

Example 25 with FileOutputStream

use of java.io.FileOutputStream in project druid by druid-io.

the class CompressionUtilsTest method testGoodGZStream.

@Test
public void testGoodGZStream() throws IOException {
    final File tmpDir = temporaryFolder.newFolder("testGoodGZStream");
    final File gzFile = new File(tmpDir, testFile.getName() + ".gz");
    Assert.assertFalse(gzFile.exists());
    CompressionUtils.gzip(new FileInputStream(testFile), new FileOutputStream(gzFile));
    Assert.assertTrue(gzFile.exists());
    try (final InputStream inputStream = new GZIPInputStream(new FileInputStream(gzFile))) {
        assertGoodDataStream(inputStream);
    }
    if (!testFile.delete()) {
        throw new IOException(String.format("Unable to delete file [%s]", testFile.getAbsolutePath()));
    }
    Assert.assertFalse(testFile.exists());
    CompressionUtils.gunzip(new FileInputStream(gzFile), testFile);
    Assert.assertTrue(testFile.exists());
    try (final InputStream inputStream = new FileInputStream(testFile)) {
        assertGoodDataStream(inputStream);
    }
}
Also used : GZIPInputStream(java.util.zip.GZIPInputStream) GZIPInputStream(java.util.zip.GZIPInputStream) FilterInputStream(java.io.FilterInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) File(java.io.File) FileInputStream(java.io.FileInputStream) Test(org.junit.Test)

Aggregations

FileOutputStream (java.io.FileOutputStream)13792 File (java.io.File)8295 IOException (java.io.IOException)6166 FileInputStream (java.io.FileInputStream)2644 OutputStream (java.io.OutputStream)2605 InputStream (java.io.InputStream)2077 BufferedOutputStream (java.io.BufferedOutputStream)1755 FileNotFoundException (java.io.FileNotFoundException)1531 OutputStreamWriter (java.io.OutputStreamWriter)1440 Test (org.junit.Test)1115 ZipEntry (java.util.zip.ZipEntry)734 BufferedWriter (java.io.BufferedWriter)668 ArrayList (java.util.ArrayList)654 ZipOutputStream (java.util.zip.ZipOutputStream)642 BufferedInputStream (java.io.BufferedInputStream)604 ByteArrayOutputStream (java.io.ByteArrayOutputStream)556 PrintWriter (java.io.PrintWriter)530 Properties (java.util.Properties)497 URL (java.net.URL)478 Writer (java.io.Writer)477