Examples with ICloseableIterator de.invesdwin.util.collections.iterable.ICloseableIterator used on opensource projects

Example 1 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<File> readRangeFilesReverse(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V lastValue = getLastValue();
        if (lastValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(lastValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeysReverse(hashKey, usedFrom.addMilliseconds(-1), to);

                private FDate delegateLastTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegateLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeysReverse();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isAfter(from)) {
                                return true;
                            } else if (element.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeKeysReverse reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateLastTime != null) {
                        time = delegateLastTime;
                        delegateLastTime = null;
                    } else if (latestLastTime != null) {
                        time = latestLastTime;
                        latestLastTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateLastTime = delegate.next();
                            if (delegateLastTime.isAfterOrEqualTo(time)) {
                                delegateLastTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 2 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValuesReverse.

protected ICloseableIterator<V> readRangeValuesReverse(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFilesReverse(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final IReverseCloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isAfter(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isAfter(from)) {
                                return true;
                            } else if (time.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isBefore(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.reverseIterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 3 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<File> readRangeFiles(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V firstValue = getFirstValue();
        if (firstValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(firstValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeys(hashKey, usedFrom.addMilliseconds(1), to);

                private FDate delegateFirstTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegateFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeys();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isBefore(from)) {
                                return true;
                            } else if (element.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeKeys reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateFirstTime != null) {
                        time = delegateFirstTime;
                        delegateFirstTime = null;
                    } else if (latestFirstTime != null) {
                        time = latestFirstTime;
                        latestFirstTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateFirstTime = delegate.next();
                            if (delegateFirstTime.isBeforeOrEqualTo(time)) {
                                delegateFirstTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 4 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class ATimeSeriesUpdater method doUpdate.

private void doUpdate() {
    final Pair<FDate, List<V>> pair = lookupTable.prepareForUpdate();
    final FDate updateFrom = pair.getFirst();
    final List<V> lastValues = pair.getSecond();
    Assertions.checkNotNull(lastValues);
    ICloseableIterable<? extends V> source = getSource(updateFrom);
    if (updateFrom != null) {
        // ensure we add no duplicate values
        source = new ASkippingIterable<V>(source) {

            @Override
            protected boolean skip(final V element) {
                return extractTime(element).isBefore(updateFrom);
            }
        };
    }
    final FlatteningIterable<? extends V> flatteningSources = new FlatteningIterable<>(lastValues, source);
    try (ICloseableIterator<? extends V> elements = flatteningSources.iterator()) {
        final ICloseableIterator<UpdateProgress> batchWriterProducer = new ICloseableIterator<UpdateProgress>() {

            @Override
            public boolean hasNext() {
                return elements.hasNext();
            }

            @Override
            public UpdateProgress next() {
                final UpdateProgress progress = new UpdateProgress();
                while (elements.hasNext()) {
                    final V element = elements.next();
                    if (progress.onElement(element)) {
                        return progress;
                    }
                }
                return progress;
            }

            @Override
            public void close() {
                elements.close();
            }
        };
        // do IO in a different thread than batch filling
        try (ACloseableIterator<UpdateProgress> batchProducer = new AProducerQueueIterator<UpdateProgress>(getClass().getSimpleName() + "_batchProducer_" + table.hashKeyToString(key), BATCH_QUEUE_SIZE) {

            @Override
            protected ICloseableIterator<ATimeSeriesUpdater<K, V>.UpdateProgress> newProducer() {
                return batchWriterProducer;
            }
        }) {
            final AtomicInteger flushIndex = new AtomicInteger();
            try (ACloseableIterator<UpdateProgress> parallelConsumer = new AParallelChunkConsumerIterator<UpdateProgress, UpdateProgress>(getClass().getSimpleName() + "_batchConsumer_" + table.hashKeyToString(key), batchProducer, BATCH_WRITER_THREADS) {

                @Override
                protected UpdateProgress doWork(final UpdateProgress request) {
                    request.write(flushIndex.incrementAndGet());
                    return request;
                }
            }) {
                while (parallelConsumer.hasNext()) {
                    final UpdateProgress progress = parallelConsumer.next();
                    count += progress.getCount();
                    if (minTime == null) {
                        minTime = progress.getMinTime();
                    }
                    maxTime = progress.getMaxTime();
                }
            }
        }
    }
}

Example 5 with ICloseableIterator

use of de.invesdwin.util.collections.iterable.ICloseableIterator in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getSegments.

private ICloseableIterable<TimeRange> getSegments(final FDate from, final FDate to, final FDate lastAvailableSegmentTo) {
    if (from == null || to == null) {
        return EmptyCloseableIterable.getInstance();
    }
    final TimeRange nextSegment = getSegmentFinder(key).query().getValue(to.addMilliseconds(1));
    final FDate adjTo;
    if (to.equalsNotNullSafe(lastAvailableSegmentTo) && nextSegment.getFrom().equalsNotNullSafe(to)) {
        // adjust for overlapping segments
        adjTo = to.addMilliseconds(-1);
    } else {
        adjTo = to;
    }
    final FDate adjFrom = from;
    final ICloseableIterable<TimeRange> segments = new ICloseableIterable<TimeRange>() {

        @Override
        public ICloseableIterator<TimeRange> iterator() {
            return new ICloseableIterator<TimeRange>() {

                private TimeRange nextSegment = getSegmentFinder(key).query().getValue(adjFrom);

                @Override
                public boolean hasNext() {
                    return nextSegment != null && nextSegment.getFrom().isBeforeOrEqualTo(adjTo);
                }

                @Override
                public TimeRange next() {
                    if (nextSegment == null) {
                        throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments nextSegment is null");
                    }
                    final TimeRange curSegment = nextSegment;
                    // get one segment later
                    nextSegment = determineNextSegment(curSegment);
                    return curSegment;
                }

                private TimeRange determineNextSegment(final TimeRange curSegment) {
                    final FDate nextSegmentStart = nextSegment.getTo().addMilliseconds(1);
                    final TimeRange nextSegment = getSegmentFinder(key).query().getValue(nextSegmentStart);
                    if (!curSegment.getTo().equalsNotNullSafe(nextSegment.getFrom()) && !nextSegmentStart.equals(nextSegment.getFrom())) {
                        // allow overlapping segments
                        throw new IllegalStateException("Segment start expected [" + curSegment.getTo() + " or " + nextSegmentStart + "] != found [" + nextSegment.getFrom() + "]");
                    }
                    return nextSegment;
                }

                @Override
                public void close() {
                    nextSegment = null;
                }
            };
        }
    };
    final ASkippingIterable<TimeRange> filteredSegments = new ASkippingIterable<TimeRange>(segments) {

        @Override
        protected boolean skip(final TimeRange element) {
            // though additionally skip ranges that exceed the available dates
            final FDate segmentTo = element.getTo();
            if (segmentTo.isBefore(adjFrom)) {
                throw new IllegalStateException("segmentTo [" + segmentTo + "] should not be before adjFrom [" + adjFrom + "]");
            }
            final FDate segmentFrom = element.getFrom();
            if (segmentFrom.isAfter(adjTo)) {
                // no need to continue going higher
                throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments end reached");
            }
            return false;
        }
    };
    return filteredSegments;
}