Examples with FDate de.invesdwin.util.time.fdate.FDate used on opensource projects

Example 1 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method prepareForUpdate.

/**
 * Deletes the last file in order to create a new updated one (so the files do not get fragmented too much between
 * updates
 */
public Pair<FDate, List<V>> prepareForUpdate() {
    final FDate latestRangeKey = storage.getFileLookupTable().getLatestRangeKey(hashKey, FDate.MAX_DATE);
    FDate updateFrom = latestRangeKey;
    final List<V> lastValues = new ArrayList<V>();
    if (latestRangeKey != null) {
        final File lastFile = newFile(latestRangeKey);
        try (SerializingCollection<V> lastColl = newSerializingCollection(lastFile)) {
            lastValues.addAll(lastColl);
        }
        // remove last value because it might be an incomplete bar
        final V lastValue = lastValues.remove(lastValues.size() - 1);
        updateFrom = extractTime.apply(lastValue);
        lastFile.delete();
        storage.getFileLookupTable().deleteRange(hashKey, latestRangeKey);
    }
    clearCaches();
    return Pair.of(updateFrom, lastValues);
}

Example 2 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<File> readRangeFilesReverse(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V lastValue = getLastValue();
        if (lastValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(lastValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeysReverse(hashKey, usedFrom.addMilliseconds(-1), to);

                private FDate delegateLastTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegateLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeysReverse();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isAfter(from)) {
                                return true;
                            } else if (element.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeKeysReverse reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateLastTime != null) {
                        time = delegateLastTime;
                        delegateLastTime = null;
                    } else if (latestLastTime != null) {
                        time = latestLastTime;
                        latestLastTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateLastTime = delegate.next();
                            if (delegateLastTime.isAfterOrEqualTo(time)) {
                                delegateLastTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 3 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValuesReverse.

protected ICloseableIterator<V> readRangeValuesReverse(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFilesReverse(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final IReverseCloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isAfter(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isAfter(from)) {
                                return true;
                            } else if (time.isBefore(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.reverseIterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isBefore(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.reverseIterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 4 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<File> readRangeFiles(final FDate from, final FDate to) {
    final FDate usedFrom;
    if (from == null) {
        final V firstValue = getFirstValue();
        if (firstValue == null) {
            return EmptyCloseableIterable.getInstance();
        }
        usedFrom = extractTime.apply(firstValue);
    } else {
        usedFrom = from;
    }
    return new ICloseableIterable<File>() {

        @Override
        public ACloseableIterator<File> iterator() {
            return new ACloseableIterator<File>() {

                // use latest time available even if delegate iterator has no values
                private FDate latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<FDate> delegate = getRangeKeys(hashKey, usedFrom.addMilliseconds(1), to);

                private FDate delegateFirstTime = null;

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegateFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<FDate> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    final ICloseableIterator<FDate> range = getAllRangeKeys();
                    return new ASkippingIterator<FDate>(range) {

                        @Override
                        protected boolean skip(final FDate element) {
                            if (element.isBefore(from)) {
                                return true;
                            } else if (element.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeKeys reached end");
                            }
                            return false;
                        }
                    };
                }

                @Override
                protected File innerNext() {
                    final FDate time;
                    if (delegateFirstTime != null) {
                        time = delegateFirstTime;
                        delegateFirstTime = null;
                    } else if (latestFirstTime != null) {
                        time = latestFirstTime;
                        latestFirstTime = null;
                        if (delegate.hasNext()) {
                            // prevent duplicate first times
                            delegateFirstTime = delegate.next();
                            if (delegateFirstTime.isBeforeOrEqualTo(time)) {
                                delegateFirstTime = null;
                            }
                        }
                    } else {
                        time = delegate.next();
                    }
                    return newFile(time);
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 5 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class ShiftUnitsRangeKeySerde method fromBytes.

@Override
public ShiftUnitsRangeKey fromBytes(final byte[] bytes) {
    final ByteBuffer buf = ByteBuffer.wrap(bytes);
    final int shiftUnits = buf.getInt();
    final FDate rangeKey = FDates.extractFDate(buf);
    return new ShiftUnitsRangeKey(shiftUnits, rangeKey);
}