Search in sources :

Example 1 with BufferedFileDataInputStream

use of de.invesdwin.util.streams.pool.buffered.BufferedFileDataInputStream in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method newResult.

private SerializingCollection<V> newResult(final String method, final MemoryFileSummary summary, final Lock readLock) {
    final TextDescription name = new TextDescription("%s[%s]: %s(%s)", ATimeSeriesUpdater.class.getSimpleName(), hashKey, method, summary);
    final File memoryFile = new File(summary.getMemoryResourceUri());
    return new SerializingCollection<V>(name, memoryFile, true) {

        @Override
        protected ISerde<V> newSerde() {
            return new ISerde<V>() {

                @Override
                public V fromBytes(final byte[] bytes) {
                    return valueSerde.fromBytes(bytes);
                }

                @Override
                public V fromBuffer(final IByteBuffer buffer, final int length) {
                    return valueSerde.fromBuffer(buffer, length);
                }

                @Override
                public int toBuffer(final IByteBuffer buffer, final V obj) {
                    throw new UnsupportedOperationException();
                }

                @Override
                public byte[] toBytes(final V obj) {
                    throw new UnsupportedOperationException();
                }
            };
        }

        @Override
        protected InputStream newFileInputStream(final File file) throws IOException {
            if (TimeseriesProperties.FILE_BUFFER_CACHE_MMAP_ENABLED) {
                readLock.lock();
                final MemoryMappedFile mmapFile = FileBufferCache.getFile(hashKey, summary.getMemoryResourceUri());
                if (mmapFile.incrementRefCount()) {
                    return new MmapInputStream(readLock, summary.newBuffer(mmapFile).asInputStream(), mmapFile);
                } else {
                    readLock.unlock();
                }
            }
            if (TimeseriesProperties.FILE_BUFFER_CACHE_SEGMENTS_ENABLED) {
                readLock.lock();
                // file buffer cache will close the file quickly
                final PreLockedBufferedFileDataInputStream in = new PreLockedBufferedFileDataInputStream(readLock, memoryFile);
                in.position(summary.getMemoryOffset());
                in.limit(summary.getMemoryOffset() + summary.getMemoryLength());
                return in;
            } else {
                // keep file input stream open as shorty as possible to prevent too many open files error
                readLock.lock();
                try (BufferedFileDataInputStream in = new BufferedFileDataInputStream(memoryFile)) {
                    in.position(summary.getMemoryOffset());
                    in.limit(summary.getMemoryOffset() + summary.getMemoryLength());
                    final PooledFastByteArrayOutputStream bos = PooledFastByteArrayOutputStream.newInstance();
                    IOUtils.copy(in, bos.asNonClosing());
                    return bos.asInputStream();
                } catch (final FileNotFoundException e) {
                    // maybe retry because of this in the outer iterator?
                    throw new RetryLaterRuntimeException("File might have been deleted in the mean time between read locks: " + file.getAbsolutePath(), e);
                } finally {
                    readLock.unlock();
                }
            }
        }

        @Override
        protected Integer getFixedLength() {
            return fixedLength;
        }

        @Override
        protected OutputStream newCompressor(final OutputStream out) {
            return storage.getCompressionFactory().newCompressor(out, ATimeSeriesUpdater.LARGE_COMPRESSOR);
        }

        @Override
        protected InputStream newDecompressor(final InputStream inputStream) {
            return storage.getCompressionFactory().newDecompressor(inputStream);
        }
    };
}
Also used : PreLockedBufferedFileDataInputStream(de.invesdwin.util.streams.pool.buffered.PreLockedBufferedFileDataInputStream) PreLockedDelegateInputStream(de.invesdwin.util.streams.PreLockedDelegateInputStream) PreLockedBufferedFileDataInputStream(de.invesdwin.util.streams.pool.buffered.PreLockedBufferedFileDataInputStream) BufferedFileDataInputStream(de.invesdwin.util.streams.pool.buffered.BufferedFileDataInputStream) InputStream(java.io.InputStream) TextDescription(de.invesdwin.util.lang.description.TextDescription) OutputStream(java.io.OutputStream) PooledFastByteArrayOutputStream(de.invesdwin.util.streams.pool.PooledFastByteArrayOutputStream) FileNotFoundException(java.io.FileNotFoundException) RetryLaterRuntimeException(de.invesdwin.context.integration.retry.RetryLaterRuntimeException) ISerde(de.invesdwin.util.marshallers.serde.ISerde) IByteBuffer(de.invesdwin.util.streams.buffer.bytes.IByteBuffer) PooledFastByteArrayOutputStream(de.invesdwin.util.streams.pool.PooledFastByteArrayOutputStream) PreLockedBufferedFileDataInputStream(de.invesdwin.util.streams.pool.buffered.PreLockedBufferedFileDataInputStream) BufferedFileDataInputStream(de.invesdwin.util.streams.pool.buffered.BufferedFileDataInputStream) ATimeSeriesUpdater(de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater) MemoryMappedFile(de.invesdwin.util.streams.buffer.MemoryMappedFile) MemoryMappedFile(de.invesdwin.util.streams.buffer.MemoryMappedFile) File(java.io.File)

Aggregations

RetryLaterRuntimeException (de.invesdwin.context.integration.retry.RetryLaterRuntimeException)1 ATimeSeriesUpdater (de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater)1 TextDescription (de.invesdwin.util.lang.description.TextDescription)1 ISerde (de.invesdwin.util.marshallers.serde.ISerde)1 PreLockedDelegateInputStream (de.invesdwin.util.streams.PreLockedDelegateInputStream)1 MemoryMappedFile (de.invesdwin.util.streams.buffer.MemoryMappedFile)1 IByteBuffer (de.invesdwin.util.streams.buffer.bytes.IByteBuffer)1 PooledFastByteArrayOutputStream (de.invesdwin.util.streams.pool.PooledFastByteArrayOutputStream)1 BufferedFileDataInputStream (de.invesdwin.util.streams.pool.buffered.BufferedFileDataInputStream)1 PreLockedBufferedFileDataInputStream (de.invesdwin.util.streams.pool.buffered.PreLockedBufferedFileDataInputStream)1 File (java.io.File)1 FileNotFoundException (java.io.FileNotFoundException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1