Examples with ICloseableIterator de.invesdwin.util.collections.iterable.ICloseableIterator used on opensource projects

Example 6 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<MemoryFileSummary> readRangeFiles(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V firstValue = getFirstValue();
                if (firstValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(firstValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFiles(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestFirstTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeys(hashKey, latestFirstTime.getRangeKey().addMilliseconds(1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).iterator();
                        final GetRangeKeysIterator rangeFiltered = new GetRangeKeysIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestFirstTime != null) {
                        summary = latestFirstTime.getValue();
                        latestFirstTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 7 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValues.

protected ICloseableIterator<V> readRangeValues(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFiles(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final ICloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isBefore(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isBefore(from)) {
                                return true;
                            } else if (time.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.iterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isAfter(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.iterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 8 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class ATimeSeriesUpdater method doUpdate.

private void doUpdate() {
    final PrepareForUpdateResult<V> prepareForUpdateResult = lookupTable.prepareForUpdate(shouldRedoLastFile());
    final FDate updateFrom = prepareForUpdateResult.getUpdateFrom();
    final List<V> lastValues = prepareForUpdateResult.getLastValues();
    final long initialAddressOffset = prepareForUpdateResult.getAddressOffset();
    final ICloseableIterable<? extends V> source = getSource(updateFrom);
    final FlatteningIterable<? extends V> flatteningSources = new FlatteningIterable<>(lastValues, source);
    try (ICloseableIterator<UpdateProgress> batchWriterProducer = new ICloseableIterator<UpdateProgress>() {

        private final UpdateProgress progress = new UpdateProgress(initialAddressOffset);

        private final ICloseableIterator<? extends V> elements = flatteningSources.iterator();

        @Override
        public boolean hasNext() {
            return elements.hasNext();
        }

        @Override
        public UpdateProgress next() {
            progress.reset();
            try {
                while (true) {
                    final V element = elements.next();
                    final FDate endTime = extractEndTime(element);
                    if (updateFrom != null) {
                        if (endTime.isBeforeNotNullSafe(updateFrom)) {
                            // ensure we add no duplicate values
                            continue;
                        }
                    }
                    if (progress.onElement(element, endTime)) {
                        return progress;
                    }
                }
            } catch (NoSuchElementException e) {
                // end reached
                if (progress.firstElement == null) {
                    throw e;
                }
            }
            return progress;
        }

        @Override
        public void close() {
            elements.close();
            progress.close();
        }
    }) {
        flush(batchWriterProducer);
    }
}

Example 9 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getSegmentsReverse.

private ICloseableIterable<TimeRange> getSegmentsReverse(final FDate from, final FDate to, final FDate lastAvailableSegmentTo) {
    if (from == null || to == null) {
        return EmptyCloseableIterable.getInstance();
    }
    final TimeRange nextSegment = getSegmentFinder(key).query().getValue(from.addMilliseconds(1));
    final FDate adjFrom;
    if (from.equalsNotNullSafe(lastAvailableSegmentTo) && nextSegment.getFrom().equalsNotNullSafe(from)) {
        // adjust for overlapping segments
        adjFrom = from.addMilliseconds(-1);
    } else {
        adjFrom = from;
    }
    final FDate adjTo = to;
    final ICloseableIterable<TimeRange> segments = new ICloseableIterable<TimeRange>() {

        @Override
        public ICloseableIterator<TimeRange> iterator() {
            return new ICloseableIterator<TimeRange>() {

                private TimeRange nextSegment = getSegmentFinder(key).query().getValue(adjFrom);

                @Override
                public boolean hasNext() {
                    return nextSegment != null && nextSegment.getTo().isAfter(adjTo);
                }

                @Override
                public TimeRange next() {
                    final TimeRange curSegment = nextSegment;
                    if (curSegment == null) {
                        throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments end reached null");
                    }
                    // get one segment earlier
                    nextSegment = getSegmentFinder(key).query().setFutureNullEnabled().getValue(nextSegment.getFrom().addMilliseconds(-1));
                    return curSegment;
                }

                @Override
                public void close() {
                    nextSegment = null;
                }
            };
        }
    };
    final ASkippingIterable<TimeRange> filteredSegments = new ASkippingIterable<TimeRange>(segments) {

        @Override
        protected boolean skip(final TimeRange element) {
            // though additionally skip ranges that exceed the available dates
            final FDate segmentTo = element.getTo();
            if (segmentTo.isBefore(adjTo)) {
                // no need to continue going lower
                throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments end reached adjTo");
            }
            // skip last value and continue with earlier ones
            final FDate segmentFrom = element.getFrom();
            return segmentFrom.isAfter(adjFrom);
        }
    };
    return filteredSegments;
}

Example 10 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<MemoryFileSummary> readRangeFilesReverse(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V lastValue = getLastValue();
                if (lastValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(lastValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFilesReverse(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestLastTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeysReverse(hashKey, latestLastTime.getRangeKey().addMilliseconds(-1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).reverseIterator();
                        final GetRangeKeysReverseIterator rangeFiltered = new GetRangeKeysReverseIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestLastTime != null) {
                        summary = latestLastTime.getValue();
                        latestLastTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}