Examples with ASkippingIterator de.invesdwin.util.collections.iterable.ASkippingIterator used on opensource projects

Example 1 with ASkippingIterator

use of de.invesdwin.util.collections.iterable.ASkippingIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<File> readRangeFilesReverse(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V lastValue = getLastValue();
        if (lastValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(lastValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeysReverse(hashKey, usedFrom.addMilliseconds(-1), to);

                private FDate delegateLastTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegateLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeysReverse();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isAfter(from)) {
                                return true;
                            } else if (element.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeKeysReverse reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateLastTime != null) {
                        time = delegateLastTime;
                        delegateLastTime = null;
                    } else if (latestLastTime != null) {
                        time = latestLastTime;
                        latestLastTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateLastTime = delegate.next();
                            if (delegateLastTime.isAfterOrEqualTo(time)) {
                                delegateLastTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 2 with ASkippingIterator

use of de.invesdwin.util.collections.iterable.ASkippingIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValuesReverse.

protected ICloseableIterator<V> readRangeValuesReverse(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFilesReverse(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final IReverseCloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isAfter(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isAfter(from)) {
                                return true;
                            } else if (time.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isBefore(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.reverseIterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 3 with ASkippingIterator

use of de.invesdwin.util.collections.iterable.ASkippingIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<File> readRangeFiles(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V firstValue = getFirstValue();
        if (firstValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(firstValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeys(hashKey, usedFrom.addMilliseconds(1), to);

                private FDate delegateFirstTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegateFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeys();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isBefore(from)) {
                                return true;
                            } else if (element.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeKeys reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateFirstTime != null) {
                        time = delegateFirstTime;
                        delegateFirstTime = null;
                    } else if (latestFirstTime != null) {
                        time = latestFirstTime;
                        latestFirstTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateFirstTime = delegate.next();
                            if (delegateFirstTime.isBeforeOrEqualTo(time)) {
                                delegateFirstTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 4 with ASkippingIterator

use of de.invesdwin.util.collections.iterable.ASkippingIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValues.

protected ICloseableIterator<V> readRangeValues(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFiles(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final ICloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isBefore(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isBefore(from)) {
                                return true;
                            } else if (time.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.iterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isAfter(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.iterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}