Examples with StandardTocWriter org.apache.nifi.provenance.toc.StandardTocWriter used on opensource projects

Example 21 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class EventFileCompressor method run.

@Override
public void run() {
    while (!shutdown) {
        File uncompressedEventFile = null;
        try {
            final long start = System.nanoTime();
            uncompressedEventFile = filesToCompress.poll(1, TimeUnit.SECONDS);
            if (uncompressedEventFile == null || shutdown) {
                continue;
            }
            File outputFile = null;
            long bytesBefore = 0L;
            StandardTocReader tocReader = null;
            File tmpTocFile = null;
            eventFileManager.obtainReadLock(uncompressedEventFile);
            try {
                StandardTocWriter tocWriter = null;
                final File tocFile = TocUtil.getTocFile(uncompressedEventFile);
                try {
                    tocReader = new StandardTocReader(tocFile);
                } catch (final IOException e) {
                    logger.error("Failed to read TOC File {}", tocFile, e);
                    continue;
                }
                bytesBefore = uncompressedEventFile.length();
                try {
                    outputFile = new File(uncompressedEventFile.getParentFile(), uncompressedEventFile.getName() + ".gz");
                    try {
                        tmpTocFile = new File(tocFile.getParentFile(), tocFile.getName() + ".tmp");
                        tocWriter = new StandardTocWriter(tmpTocFile, true, false);
                        compress(uncompressedEventFile, tocReader, outputFile, tocWriter);
                        tocWriter.close();
                    } catch (final IOException ioe) {
                        logger.error("Failed to compress {} on rollover", uncompressedEventFile, ioe);
                    }
                } finally {
                    CloseableUtil.closeQuietly(tocReader, tocWriter);
                }
            } finally {
                eventFileManager.releaseReadLock(uncompressedEventFile);
            }
            eventFileManager.obtainWriteLock(uncompressedEventFile);
            try {
                // Attempt to delete the input file and associated toc file
                if (uncompressedEventFile.delete()) {
                    if (tocReader != null) {
                        final File tocFile = tocReader.getFile();
                        if (!tocFile.delete()) {
                            logger.warn("Failed to delete {}; this file should be cleaned up manually", tocFile);
                        }
                        if (tmpTocFile != null) {
                            tmpTocFile.renameTo(tocFile);
                        }
                    }
                } else {
                    logger.warn("Failed to delete {}; this file should be cleaned up manually", uncompressedEventFile);
                }
            } finally {
                eventFileManager.releaseWriteLock(uncompressedEventFile);
            }
            final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            final long bytesAfter = outputFile.length();
            final double reduction = 100 * (1 - (double) bytesAfter / (double) bytesBefore);
            final String reductionTwoDecimals = String.format("%.2f", reduction);
            logger.debug("Successfully compressed Provenance Event File {} in {} millis from {} to {}, a reduction of {}%", uncompressedEventFile, millis, FormatUtils.formatDataSize(bytesBefore), FormatUtils.formatDataSize(bytesAfter), reductionTwoDecimals);
        } catch (final InterruptedException e) {
            Thread.currentThread().interrupt();
            return;
        } catch (final Exception e) {
            logger.error("Failed to compress {}", uncompressedEventFile, e);
        }
    }
}