Examples with NonCloseableOutputStream org.apache.nifi.stream.io.NonCloseableOutputStream used on opensource projects

Example 1 with NonCloseableOutputStream

use of org.apache.nifi.stream.io.NonCloseableOutputStream in project nifi by apache.

the class CompressableRecordWriter method resetWriteStream.

/**
 * Resets the streams to prepare for a new block
 *
 * @param eventId the first id that will be written to the new block
 * @throws IOException if unable to flush/close the current streams properly
 */
protected void resetWriteStream(final Long eventId) throws IOException {
    try {
        if (out != null) {
            out.flush();
        }
        final long byteOffset = (byteCountingOut == null) ? rawOutStream.getBytesWritten() : byteCountingOut.getBytesWritten();
        final TocWriter tocWriter = getTocWriter();
        if (compressed) {
            // We don't have to check if the writer is dirty because we will have already checked before calling this method.
            if (out != null) {
                out.close();
            }
            if (tocWriter != null && eventId != null) {
                tocWriter.addBlockOffset(rawOutStream.getBytesWritten(), eventId);
            }
            final OutputStream writableStream = new BufferedOutputStream(new GZIPOutputStream(new NonCloseableOutputStream(rawOutStream), 1), 65536);
            this.byteCountingOut = new ByteCountingOutputStream(writableStream, byteOffset);
        } else {
            if (tocWriter != null && eventId != null) {
                tocWriter.addBlockOffset(rawOutStream.getBytesWritten(), eventId);
            }
            this.byteCountingOut = rawOutStream;
        }
        this.out = new DataOutputStream(byteCountingOut);
        resetDirtyFlag();
    } catch (final IOException ioe) {
        markDirty();
        throw ioe;
    }
}

Example 2 with NonCloseableOutputStream

use of org.apache.nifi.stream.io.NonCloseableOutputStream in project nifi by apache.

the class EventFileCompressor method compress.

public static void compress(final File input, final TocReader tocReader, final File output, final TocWriter tocWriter) throws IOException {
    try (final InputStream fis = new FileInputStream(input);
        final OutputStream fos = new FileOutputStream(output);
        final ByteCountingOutputStream byteCountingOut = new ByteCountingOutputStream(fos)) {
        int blockIndex = 0;
        while (true) {
            // Determine the min and max byte ranges for the current block.
            final long blockStart = tocReader.getBlockOffset(blockIndex);
            if (blockStart == -1) {
                break;
            }
            long blockEnd = tocReader.getBlockOffset(blockIndex + 1);
            if (blockEnd < 0) {
                blockEnd = input.length();
            }
            final long firstEventId = tocReader.getFirstEventIdForBlock(blockIndex);
            final long blockStartOffset = byteCountingOut.getBytesWritten();
            try (final OutputStream ncos = new NonCloseableOutputStream(byteCountingOut);
                final OutputStream gzipOut = new GZIPOutputStream(ncos, 1)) {
                StreamUtils.copy(fis, gzipOut, blockEnd - blockStart);
            }
            tocWriter.addBlockOffset(blockStartOffset, firstEventId);
            blockIndex++;
        }
    }
    // Close the TOC Reader and TOC Writer
    CloseableUtil.closeQuietly(tocReader, tocWriter);
}