Examples with MemoryFileSummary de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary used on opensource projects

Example 6 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method getLastValue.

public V getLastValue() {
    if (cachedLastValue == null) {
        final MemoryFileSummary latestValue = storage.getFileLookupTable().getLatestValue(hashKey, FDate.MAX_DATE);
        final V lastValue;
        if (latestValue == null) {
            lastValue = null;
        } else {
            lastValue = latestValue.getLastValue(valueSerde);
        }
        cachedLastValue = Optional.ofNullable(lastValue);
    }
    return cachedLastValue.orElse(null);
}

Example 7 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getPreviousValue.

public V getPreviousValue(final FDate date, final int shiftBackUnits) {
    assertShiftUnitsPositiveNonZero(shiftBackUnits);
    final V firstValue = getFirstValue();
    final FDate firstTime = segmentedTable.extractEndTime(firstValue);
    if (date.isBeforeOrEqualTo(firstTime)) {
        return firstValue;
    } else {
        final SingleValue value = storage.getOrLoad_previousValueLookupTable(hashKey, date, shiftBackUnits, () -> {
            final MutableReference<V> previousValue = new MutableReference<>();
            final MutableInt shiftBackRemaining = new MutableInt(shiftBackUnits);
            try (ICloseableIterator<V> rangeValuesReverse = readRangeValuesReverse(date, null, DisabledLock.INSTANCE, new ISkipFileFunction() {

                @Override
                public boolean skipFile(final MemoryFileSummary file) {
                    final boolean skip = previousValue.get() != null && file.getValueCount() < shiftBackRemaining.intValue();
                    if (skip) {
                        shiftBackRemaining.add(file.getValueCount());
                    }
                    return skip;
                }
            }).iterator()) {
                while (shiftBackRemaining.intValue() >= 0) {
                    previousValue.set(rangeValuesReverse.next());
                    shiftBackRemaining.decrement();
                }
            } catch (final NoSuchElementException e) {
            // ignore
            }
            return new SingleValue(valueSerde, previousValue.get());
        });
        return value.getValue(valueSerde);
    }
}

Example 8 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getFirstValue.

public V getFirstValue() {
    if (cachedFirstValue != null) {
        maybePrepareForUpdate(null);
    }
    if (cachedFirstValue == null) {
        final FDate firstAvailableSegmentFrom = getFirstAvailableSegmentFrom(key);
        if (firstAvailableSegmentFrom == null) {
            cachedFirstValue = Optional.empty();
        } else {
            FDate lastAvailableSegmentTo = getLastAvailableSegmentTo(key, null);
            final TimeRange nextSegment = getSegmentFinder(key).query().getValue(lastAvailableSegmentTo.addMilliseconds(1));
            if (nextSegment.getFrom().equals(lastAvailableSegmentTo)) {
                // adjust for overlapping segments
                lastAvailableSegmentTo = lastAvailableSegmentTo.addMilliseconds(-1);
            }
            final IHistoricalCacheQuery<TimeRange> segmentFinderQuery = getSegmentFinder(key).query();
            final TimeRange lastSegment = segmentFinderQuery.getValue(lastAvailableSegmentTo);
            TimeRange segment = segmentFinderQuery.getValue(firstAvailableSegmentFrom);
            if (!segment.getFrom().equalsNotNullSafe(firstAvailableSegmentFrom)) {
                throw new IllegalStateException("segment.from [" + segment.getFrom() + "] should be equal to firstAvailableSegmentFrom [" + firstAvailableSegmentFrom + "]");
            }
            while (cachedFirstValue == null && segment.getFrom().isBeforeOrEqualTo(lastSegment.getFrom())) {
                final SegmentedKey<K> segmentedKey = new SegmentedKey<K>(key, segment);
                maybeInitSegment(segmentedKey);
                final String segmentedHashKey = segmentedTable.hashKeyToString(segmentedKey);
                final MemoryFileSummary latestValue = storage.getFileLookupTable().getLatestValue(segmentedHashKey, FDate.MIN_DATE);
                final V firstValue;
                if (latestValue == null) {
                    segment = segmentFinderQuery.getValue(segment.getTo().addMilliseconds(1));
                } else {
                    firstValue = latestValue.getFirstValue(valueSerde);
                    cachedFirstValue = Optional.of(firstValue);
                }
            }
            if (cachedFirstValue == null) {
                cachedFirstValue = Optional.empty();
            }
        }
    }
    return cachedFirstValue.orElse(null);
}

Example 9 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method getFirstValue.

public V getFirstValue() {
    if (cachedFirstValue == null) {
        final MemoryFileSummary latestValue = storage.getFileLookupTable().getLatestValue(hashKey, FDate.MIN_DATE);
        final V firstValue;
        if (latestValue == null) {
            firstValue = null;
        } else {
            firstValue = latestValue.getFirstValue(valueSerde);
        }
        cachedFirstValue = Optional.ofNullable(firstValue);
    }
    return cachedFirstValue.orElse(null);
}

Example 10 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method prepareForUpdate.

/**
 * When shouldRedoLastFile=true this deletes the last file in order to create a new updated one (so the files do not
 * get fragmented too much between updates
 */
public synchronized PrepareForUpdateResult<V> prepareForUpdate(final boolean shouldRedoLastFile) {
    final RangeTableRow<String, FDate, MemoryFileSummary> latestFile = storage.getFileLookupTable().getLatest(hashKey, FDate.MAX_DATE);
    final FDate updateFrom;
    final List<V> lastValues;
    final long addressOffset;
    if (latestFile != null) {
        final FDate latestRangeKey;
        final MemoryFileSummary latestSummary = latestFile.getValue();
        if (shouldRedoLastFile && latestSummary.getValueCount() < ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
            lastValues = new ArrayList<V>();
            try (ICloseableIterator<V> lastColl = newResult("prepareForUpdate", latestSummary, DisabledLock.INSTANCE).iterator()) {
                Lists.toListWithoutHasNext(lastColl, lastValues);
            }
            if (!lastValues.isEmpty()) {
                // remove last value because it might be an incomplete bar
                final V lastValue = lastValues.remove(lastValues.size() - 1);
                addressOffset = latestSummary.getMemoryOffset();
                updateFrom = extractEndTime.apply(lastValue);
                latestRangeKey = latestFile.getRangeKey();
            } else {
                addressOffset = latestSummary.getMemoryOffset() + latestSummary.getMemoryLength() + 1L;
                updateFrom = latestFile.getRangeKey();
                latestRangeKey = latestFile.getRangeKey().addMilliseconds(1);
            }
        } else {
            lastValues = Collections.emptyList();
            addressOffset = latestSummary.getMemoryOffset() + latestSummary.getMemoryLength() + 1L;
            updateFrom = latestFile.getRangeKey();
            latestRangeKey = latestFile.getRangeKey().addMilliseconds(1);
        }
        storage.getFileLookupTable().deleteRange(hashKey, latestRangeKey);
        storage.deleteRange_latestValueLookupTable(hashKey, latestRangeKey);
        // we cannot be sure here about the date since shift keys can be arbitrarily large
        storage.deleteRange_nextValueLookupTable(hashKey);
        storage.deleteRange_previousValueLookupTable(hashKey, latestRangeKey);
    } else {
        updateFrom = null;
        lastValues = Collections.emptyList();
        addressOffset = 0L;
    }
    clearCaches();
    return new PrepareForUpdateResult<>(updateFrom, lastValues, addressOffset);
}