Examples with RangeTableRow ezdb.table.RangeTableRow used on opensource projects

Example 1 with RangeTableRow

use of ezdb.table.RangeTableRow in project invesdwin-context-persistence by subes.

the class TestStockData method assertIteration.

private void assertIteration(final int countFDates, final FDate fromFDate, final FDate toFDate) {
    TableIterator<RangeTableRow<String, FDate, Integer>> range = table.range(MSFT, fromFDate, toFDate);
    int iteratedBars = 0;
    int prevValue = 0;
    FDate left1000FDate = null;
    FDate left900FDate = null;
    final Instant start = new Instant();
    while (range.hasNext()) {
        final RangeTableRow<String, FDate, Integer> next = range.next();
        final Integer value = next.getValue();
        // System.out.println(value);
        iteratedBars++;
        Assertions.checkTrue(prevValue < value);
        prevValue = value;
        if (iteratedBars == countFDates - 999) {
            left1000FDate = next.getRangeKey();
        }
        if (iteratedBars == countFDates - 900) {
            left900FDate = next.getRangeKey();
        }
    }
    System.out.println("took: " + start);
    Assertions.checkEquals(countFDates, iteratedBars);
    Assertions.checkEquals(1, table.getLatest(MSFT, fromFDate).getValue());
    Assertions.checkEquals(countFDates, table.getLatest(MSFT, toFDate).getValue());
    // System.out.println(left1000FDate +" -> "+left900FDate);
    range = table.range(MSFT, left1000FDate, left900FDate);
    int curLeftIt = 0;
    RangeTableRow<String, FDate, Integer> prev = null;
    while (range.hasNext()) {
        final RangeTableRow<String, FDate, Integer> next = range.next();
        curLeftIt++;
        Assertions.checkEquals(countFDates - 1000 + curLeftIt, next.getValue());
        if (prev != null) {
            final Integer nextFromPrevPlus = table.getNext(MSFT, new FDate(prev.getRangeKey().millisValue() + 1)).getValue();
            Assertions.checkEquals(next.getValue(), nextFromPrevPlus);
            final Integer prevFromNextMinus = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue() - 1)).getValue();
            Assertions.checkEquals(prev.getValue(), prevFromNextMinus);
        }
        final Integer nextFromNextIsSame = table.getNext(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), nextFromNextIsSame);
        final Integer prevFromNextIsSame = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), prevFromNextIsSame);
        prev = next;
    }
    Assertions.checkEquals(100, curLeftIt);
}

Example 2 with RangeTableRow

use of ezdb.table.RangeTableRow in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<MemoryFileSummary> readRangeFiles(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V firstValue = getFirstValue();
                if (firstValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(firstValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFiles(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestFirstTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeys(hashKey, latestFirstTime.getRangeKey().addMilliseconds(1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).iterator();
                        final GetRangeKeysIterator rangeFiltered = new GetRangeKeysIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestFirstTime != null) {
                        summary = latestFirstTime.getValue();
                        latestFirstTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 3 with RangeTableRow

use of ezdb.table.RangeTableRow in project invesdwin-context-persistence by subes.

the class TestStockData method testStockData.

@Test
public void testStockData() throws IOException, ParseException {
    final InputStream in = new ClassPathResource("MSFT.txt", getClass()).getInputStream();
    final List<String> lines = CharStreams.readLines(new InputStreamReader(in));
    lines.remove(0);
    lines.remove(0);
    Collections.reverse(lines);
    in.close();
    Long prevLongTime = null;
    int countFDate = 0;
    FDate firstFDate = null;
    FDate lastFDate = null;
    for (final String line : lines) {
        final String[] split = line.split(",");
        Assertions.checkEquals(7, split.length);
        countFDate++;
        final String dateStr = split[0];
        final FDate date = FDate.valueOf(dateStr, "yyyy-MM-dd");
        if (firstFDate == null) {
            firstFDate = date;
        }
        lastFDate = date;
        final long longTime = date.millisValue();
        if (prevLongTime != null) {
            // System.out.println(dateStr + ":"+date + " - "+prevLongTime+"  < " + longTime + " -> "
            // + (prevLongTime < longTime));
            Assertions.checkTrue(prevLongTime < longTime);
        }
        table.put(MSFT, date, countFDate);
        prevLongTime = longTime;
    }
    System.out.println(MSFT + " has " + countFDate + " bars");
    assertIteration(countFDate, MIN_DATE, MAX_DATE);
    assertIteration(countFDate, firstFDate, lastFDate);
    // Fri Jan 24 23:46:40 UTC 2014
    TableIterator<RangeTableRow<String, FDate, Integer>> range = table.range(MSFT, FDateBuilder.newDate(2014, 1, 23));
    int countBars = 0;
    while (range.hasNext()) {
        final RangeTableRow<String, FDate, Integer> next = range.next();
        System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(253, countBars);
    range = table.range(MSFT, FDateBuilder.newDate(2014, 1, 23), null);
    countBars = 0;
    while (range.hasNext()) {
        final RangeTableRow<String, FDate, Integer> next = range.next();
        // System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(253, countBars);
    range = table.range(MSFT, null, FDateBuilder.newDate(1987, 1, 1));
    countBars = 0;
    while (range.hasNext()) {
        final RangeTableRow<String, FDate, Integer> next = range.next();
        // System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(204, countBars);
}

Example 4 with RangeTableRow

use of ezdb.table.RangeTableRow in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method deleteAll.

public synchronized void deleteAll() {
    final ADelegateRangeTable<String, TimeRange, SegmentStatus> segmentStatusTable = storage.getSegmentStatusTable();
    final List<TimeRange> rangeKeys;
    try (ICloseableIterator<TimeRange> rangeKeysIterator = new ATransformingIterator<RangeTableRow<String, TimeRange, SegmentStatus>, TimeRange>(segmentStatusTable.range(hashKey)) {

        @Override
        protected TimeRange transform(final RangeTableRow<String, TimeRange, SegmentStatus> value) {
            return value.getRangeKey();
        }
    }) {
        rangeKeys = Lists.toListWithoutHasNext(rangeKeysIterator);
    }
    for (int i = 0; i < rangeKeys.size(); i++) {
        final TimeRange rangeKey = rangeKeys.get(i);
        segmentedTable.deleteRange(new SegmentedKey<K>(key, rangeKey));
    }
    segmentStatusTable.deleteRange(hashKey);
    storage.deleteRange_latestValueLookupTable(hashKey);
    storage.deleteRange_nextValueLookupTable(hashKey);
    storage.deleteRange_previousValueLookupTable(hashKey);
    clearCaches();
}

Example 5 with RangeTableRow

use of ezdb.table.RangeTableRow in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFilesReverse.

protected ICloseableIterable<MemoryFileSummary> readRangeFilesReverse(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V lastValue = getLastValue();
                if (lastValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(lastValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFilesReverse(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestLastTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                // add 1 ms to not collide with firstTime
                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestLastTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeysReverse(hashKey, latestLastTime.getRangeKey().addMilliseconds(-1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestLastTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeysReverse(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).reverseIterator();
                        final GetRangeKeysReverseIterator rangeFiltered = new GetRangeKeysReverseIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestLastTime != null) {
                        summary = latestLastTime.getValue();
                        latestLastTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}