Examples with FlatteningIterator de.invesdwin.util.collections.iterable.FlatteningIterator used on opensource projects

Example 1 with FlatteningIterator

use of de.invesdwin.util.collections.iterable.FlatteningIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValuesReverse.

protected ICloseableIterator<V> readRangeValuesReverse(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFilesReverse(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final IReverseCloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isAfter(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isAfter(from)) {
                                return true;
                            } else if (time.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isBefore(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.reverseIterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 2 with FlatteningIterator

use of de.invesdwin.util.collections.iterable.FlatteningIterator in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValues.

protected ICloseableIterator<V> readRangeValues(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFiles(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final ICloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isBefore(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isBefore(from)) {
                                return true;
                            } else if (time.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.iterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isAfter(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.iterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}