Examples with TextDescription de.invesdwin.util.lang.description.TextDescription used on opensource projects

Example 1 with TextDescription

use of de.invesdwin.util.lang.description.TextDescription in project invesdwin-context-persistence by subes.

the class FileLiveSegment method newSerializingCollection.

private SerializingCollection<V> newSerializingCollection() {
    final File file = getFile();
    Files.deleteQuietly(file);
    try {
        Files.forceMkdirParent(file);
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
    final TextDescription name = new TextDescription("%s[%s]: newSerializingCollection()", FileLiveSegment.class.getSimpleName(), segmentedKey);
    return new SerializingCollection<V>(name, file, false) {

        @Override
        protected ISerde<V> newSerde() {
            return historicalSegmentTable.newValueSerde();
        }

        @Override
        protected Integer getFixedLength() {
            return historicalSegmentTable.newValueFixedLength();
        }

        @Override
        protected OutputStream newCompressor(final OutputStream out) {
            return compressionFactory.newCompressor(out, LARGE_COMPRESSOR);
        }

        @Override
        protected InputStream newDecompressor(final InputStream inputStream) {
            return compressionFactory.newDecompressor(inputStream);
        }

        @Override
        protected InputStream newFileInputStream(final File file) throws IOException {
            throw new UnsupportedOperationException("use getFlushedValues() instead");
        }
    };
}

Example 2 with TextDescription

use of de.invesdwin.util.lang.description.TextDescription in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<MemoryFileSummary> readRangeFiles(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V firstValue = getFirstValue();
                if (firstValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(firstValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFiles(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestFirstTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeys(hashKey, latestFirstTime.getRangeKey().addMilliseconds(1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).iterator();
                        final GetRangeKeysIterator rangeFiltered = new GetRangeKeysIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestFirstTime != null) {
                        summary = latestFirstTime.getValue();
                        latestFirstTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 3 with TextDescription

use of de.invesdwin.util.lang.description.TextDescription in project invesdwin-context-persistence by subes.

the class SerializingCollectionTest method testSymlinks.

@Test
public void testSymlinks() throws IOException {
    final File file = new File(ContextProperties.TEMP_DIRECTORY, "testSymlinks.bin.lz4");
    final SerializingCollection<String> writer = new SerializingCollection<>(new TextDescription("%s", SerializingCollectionTest.class.getSimpleName()), file, false);
    for (int i = 0; i < 100; i++) {
        writer.add("asdf" + i);
    }
    writer.close();
    final File symlink = new File(ContextProperties.TEMP_DIRECTORY, file.getName() + "_symlink");
    Files.createSymbolicLink(symlink.getAbsoluteFile().toPath(), file.getAbsoluteFile().toPath());
    final SerializingCollection<String> reader = new SerializingCollection<>(new TextDescription("%s", SerializingCollectionTest.class.getSimpleName()), symlink, true);
    final ICloseableIterator<String> iterator = reader.iterator();
    for (int i = 0; i < 100; i++) {
        Assertions.checkEquals("asdf" + i, iterator.next());
    }
    iterator.close();
    reader.close();
}

Example 4 with TextDescription

use of de.invesdwin.util.lang.description.TextDescription in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<MemoryFileSummary> readRangeFilesReverse(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V lastValue = getLastValue();
                if (lastValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(lastValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFilesReverse(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestLastTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeysReverse(hashKey, latestLastTime.getRangeKey().addMilliseconds(-1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).reverseIterator();
                        final GetRangeKeysReverseIterator rangeFiltered = new GetRangeKeysReverseIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestLastTime != null) {
                        summary = latestLastTime.getValue();
                        latestLastTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 5 with TextDescription

use of de.invesdwin.util.lang.description.TextDescription in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method newResult.

private SerializingCollection<V> newResult(final String method, final MemoryFileSummary summary, final Lock readLock) {
    final TextDescription name = new TextDescription("%s[%s]: %s(%s)", ATimeSeriesUpdater.class.getSimpleName(), hashKey, method, summary);
    final File memoryFile = new File(summary.getMemoryResourceUri());
    return new SerializingCollection<V>(name, memoryFile, true) {

        @Override
        protected ISerde<V> newSerde() {
            return new ISerde<V>() {

                @Override
                public V fromBytes(final byte[] bytes) {
                    return valueSerde.fromBytes(bytes);
                }

                @Override
                public V fromBuffer(final IByteBuffer buffer, final int length) {
                    return valueSerde.fromBuffer(buffer, length);
                }

                @Override
                public int toBuffer(final IByteBuffer buffer, final V obj) {
                    throw new UnsupportedOperationException();
                }

                @Override
                public byte[] toBytes(final V obj) {
                    throw new UnsupportedOperationException();
                }
            };
        }

        @Override
        protected InputStream newFileInputStream(final File file) throws IOException {
            if (TimeseriesProperties.FILE_BUFFER_CACHE_MMAP_ENABLED) {
                readLock.lock();
                final MemoryMappedFile mmapFile = FileBufferCache.getFile(hashKey, summary.getMemoryResourceUri());
                if (mmapFile.incrementRefCount()) {
                    return new MmapInputStream(readLock, summary.newBuffer(mmapFile).asInputStream(), mmapFile);
                } else {
                    readLock.unlock();
                }
            }
            if (TimeseriesProperties.FILE_BUFFER_CACHE_SEGMENTS_ENABLED) {
                readLock.lock();
                // file buffer cache will close the file quickly
                final PreLockedBufferedFileDataInputStream in = new PreLockedBufferedFileDataInputStream(readLock, memoryFile);
                in.position(summary.getMemoryOffset());
                in.limit(summary.getMemoryOffset() + summary.getMemoryLength());
                return in;
            } else {
                // keep file input stream open as shorty as possible to prevent too many open files error
                readLock.lock();
                try (BufferedFileDataInputStream in = new BufferedFileDataInputStream(memoryFile)) {
                    in.position(summary.getMemoryOffset());
                    in.limit(summary.getMemoryOffset() + summary.getMemoryLength());
                    final PooledFastByteArrayOutputStream bos = PooledFastByteArrayOutputStream.newInstance();
                    IOUtils.copy(in, bos.asNonClosing());
                    return bos.asInputStream();
                } catch (final FileNotFoundException e) {
                    // maybe retry because of this in the outer iterator?
                    throw new RetryLaterRuntimeException("File might have been deleted in the mean time between read locks: " + file.getAbsolutePath(), e);
                } finally {
                    readLock.unlock();
                }
            }
        }

        @Override
        protected Integer getFixedLength() {
            return fixedLength;
        }

        @Override
        protected OutputStream newCompressor(final OutputStream out) {
            return storage.getCompressionFactory().newCompressor(out, ATimeSeriesUpdater.LARGE_COMPRESSOR);
        }

        @Override
        protected InputStream newDecompressor(final InputStream inputStream) {
            return storage.getCompressionFactory().newDecompressor(inputStream);
        }
    };
}