Examples with ICloseableIterable de.invesdwin.util.collections.iterable.ICloseableIterable used on opensource projects

Example 6 with ICloseableIterable

use of de.invesdwin.util.collections.iterable.ICloseableIterable in project invesdwin-context-persistence by subes.

the class ATimeSeriesDBTest method testGetPreviousAndNextSkipFile.

@Test
public void testGetPreviousAndNextSkipFile() throws IncompleteUpdateFoundException {
    final String key = "asdf";
    final ATimeSeriesDB<String, FDate> table = new ATimeSeriesDB<String, FDate>("testGetPreviousSkipFile") {

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }
    };
    final List<FDate> dates = new ArrayList<>();
    for (int i = 0; i < 100_000; i++) {
        dates.add(new FDate(i));
    }
    final MutableInt segments = new MutableInt();
    new ATimeSeriesUpdater<String, FDate>(key, table) {

        @Override
        protected ICloseableIterable<? extends FDate> getSource(final FDate updateFrom) {
            return WrapperCloseableIterable.maybeWrap(dates);
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final FDate element) {
            return element;
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<String, FDate>.UpdateProgress updateProgress) {
            segments.increment();
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    }.update();
    Assertions.assertThat(segments.intValue()).isEqualByComparingTo(10);
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, dates.get(dates.size() - 1), i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, FDate.MAX_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(0);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, FDate.MIN_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(i);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, dates.get(0), i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(i);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, FDate.MIN_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, FDate.MAX_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
}

Example 7 with ICloseableIterable

use of de.invesdwin.util.collections.iterable.ICloseableIterable in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesDBWithUnlimitedCacheTest method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    final AHistoricalCache<TimeRange> segmentFinder = PeriodicalSegmentFinder.newCache(new Duration(2, FTimeUnit.YEARS), false);
    table = new ASegmentedTimeSeriesDB<String, FDate>(getClass().getSimpleName()) {

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final String key) {
            return segmentFinder;
        }

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }

        @Override
        protected ICloseableIterable<? extends FDate> downloadSegmentElements(final SegmentedKey<String> segmentedKey) {
            return new ASkippingIterable<FDate>(WrapperCloseableIterable.maybeWrap(entities)) {

                private final FDate from = segmentedKey.getSegment().getFrom();

                private final FDate to = segmentedKey.getSegment().getTo();

                @Override
                protected boolean skip(final FDate element) {
                    return element.isBefore(from) || element.isAfter(to);
                }
            };
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final String key) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(0)).getFrom();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final String key, final FDate updateTo) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(entities.size() - 1)).getTo();
        }

        @Override
        protected String getElementsName() {
            return "values";
        }
    };
}

Example 8 with ICloseableIterable

use of de.invesdwin.util.collections.iterable.ICloseableIterable in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesDBWithNoCacheTest method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    final AHistoricalCache<TimeRange> segmentFinder = PeriodicalSegmentFinder.newCache(new Duration(2, FTimeUnit.YEARS), false);
    table = new ASegmentedTimeSeriesDB<String, FDate>(getClass().getSimpleName()) {

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final String key) {
            return segmentFinder;
        }

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }

        @Override
        protected ICloseableIterable<? extends FDate> downloadSegmentElements(final SegmentedKey<String> segmentedKey) {
            return new ASkippingIterable<FDate>(WrapperCloseableIterable.maybeWrap(entities)) {

                private final FDate from = segmentedKey.getSegment().getFrom();

                private final FDate to = segmentedKey.getSegment().getTo();

                @Override
                protected boolean skip(final FDate element) {
                    return element.isBefore(from) || element.isAfter(to);
                }
            };
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final String key) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(0)).getFrom();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final String key, final FDate updateTo) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(entities.size() - 1)).getTo();
        }

        @Override
        protected String getElementsName() {
            return "values";
        }
    };
}

Example 9 with ICloseableIterable

use of de.invesdwin.util.collections.iterable.ICloseableIterable in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesDBWithLimitedCacheTest method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    final AHistoricalCache<TimeRange> segmentFinder = PeriodicalSegmentFinder.newCache(new Duration(2, FTimeUnit.YEARS), false);
    table = new ASegmentedTimeSeriesDB<String, FDate>(getClass().getSimpleName()) {

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final String key) {
            return segmentFinder;
        }

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }

        @Override
        protected ICloseableIterable<? extends FDate> downloadSegmentElements(final SegmentedKey<String> segmentedKey) {
            return new ASkippingIterable<FDate>(WrapperCloseableIterable.maybeWrap(entities)) {

                private final FDate from = segmentedKey.getSegment().getFrom();

                private final FDate to = segmentedKey.getSegment().getTo();

                @Override
                protected boolean skip(final FDate element) {
                    return element.isBefore(from) || element.isAfter(to);
                }
            };
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final String key) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(0)).getFrom();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final String key, final FDate updateTo) {
            if (entities.isEmpty()) {
                return null;
            }
            return segmentFinder.query().getValue(entities.get(entities.size() - 1)).getTo();
        }

        @Override
        protected String getElementsName() {
            return "values";
        }
    };
}

Example 10 with ICloseableIterable

use of de.invesdwin.util.collections.iterable.ICloseableIterable in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getSegments.

private ICloseableIterable<TimeRange> getSegments(final FDate from, final FDate to, final FDate lastAvailableSegmentTo) {
    if (from == null || to == null) {
        return EmptyCloseableIterable.getInstance();
    }
    final TimeRange nextSegment = getSegmentFinder(key).query().getValue(to.addMilliseconds(1));
    final FDate adjTo;
    if (to.equalsNotNullSafe(lastAvailableSegmentTo) && nextSegment.getFrom().equalsNotNullSafe(to)) {
        // adjust for overlapping segments
        adjTo = to.addMilliseconds(-1);
    } else {
        adjTo = to;
    }
    final FDate adjFrom = from;
    final ICloseableIterable<TimeRange> segments = new ICloseableIterable<TimeRange>() {

        @Override
        public ICloseableIterator<TimeRange> iterator() {
            return new ICloseableIterator<TimeRange>() {

                private TimeRange nextSegment = getSegmentFinder(key).query().getValue(adjFrom);

                @Override
                public boolean hasNext() {
                    return nextSegment != null && nextSegment.getFrom().isBeforeOrEqualTo(adjTo);
                }

                @Override
                public TimeRange next() {
                    if (nextSegment == null) {
                        throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments nextSegment is null");
                    }
                    final TimeRange curSegment = nextSegment;
                    // get one segment later
                    nextSegment = determineNextSegment(curSegment);
                    return curSegment;
                }

                private TimeRange determineNextSegment(final TimeRange curSegment) {
                    final FDate nextSegmentStart = nextSegment.getTo().addMilliseconds(1);
                    final TimeRange nextSegment = getSegmentFinder(key).query().getValue(nextSegmentStart);
                    if (!curSegment.getTo().equalsNotNullSafe(nextSegment.getFrom()) && !nextSegmentStart.equals(nextSegment.getFrom())) {
                        // allow overlapping segments
                        throw new IllegalStateException("Segment start expected [" + curSegment.getTo() + " or " + nextSegmentStart + "] != found [" + nextSegment.getFrom() + "]");
                    }
                    return nextSegment;
                }

                @Override
                public void close() {
                    nextSegment = null;
                }
            };
        }
    };
    final ASkippingIterable<TimeRange> filteredSegments = new ASkippingIterable<TimeRange>(segments) {

        @Override
        protected boolean skip(final TimeRange element) {
            // though additionally skip ranges that exceed the available dates
            final FDate segmentTo = element.getTo();
            if (segmentTo.isBefore(adjFrom)) {
                throw new IllegalStateException("segmentTo [" + segmentTo + "] should not be before adjFrom [" + adjFrom + "]");
            }
            final FDate segmentFrom = element.getFrom();
            if (segmentFrom.isAfter(adjTo)) {
                // no need to continue going higher
                throw new FastNoSuchElementException("ASegmentedTimeSeriesStorageCache getSegments end reached");
            }
            return false;
        }
    };
    return filteredSegments;
}