Examples with MemoryFileSummary de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary used on opensource projects

Example 1 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class ASegmentedTimeSeriesStorageCache method getNextValue.

public V getNextValue(final FDate date, final int shiftForwardUnits) {
    assertShiftUnitsPositiveNonZero(shiftForwardUnits);
    final V lastValue = getLastValue();
    final FDate lastTime = segmentedTable.extractEndTime(lastValue);
    if (date.isAfterOrEqualTo(lastTime)) {
        return lastValue;
    } else {
        final SingleValue value = storage.getOrLoad_nextValueLookupTable(hashKey, date, shiftForwardUnits, () -> {
            final MutableReference<V> nextValue = new MutableReference<>();
            final MutableInt shiftForwardRemaining = new MutableInt(shiftForwardUnits);
            try (ICloseableIterator<V> rangeValues = readRangeValues(date, null, DisabledLock.INSTANCE, new ISkipFileFunction() {

                @Override
                public boolean skipFile(final MemoryFileSummary file) {
                    final boolean skip = nextValue.get() != null && file.getValueCount() < shiftForwardRemaining.intValue();
                    if (skip) {
                        shiftForwardRemaining.subtract(file.getValueCount());
                    }
                    return skip;
                }
            }).iterator()) {
                while (shiftForwardRemaining.intValue() >= 0) {
                    nextValue.set(rangeValues.next());
                    shiftForwardRemaining.decrement();
                }
            } catch (final NoSuchElementException e) {
            // ignore
            }
            return new SingleValue(valueSerde, nextValue.get());
        });
        return value.getValue(valueSerde);
    }
}

Example 2 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeFiles.

protected ICloseableIterable<MemoryFileSummary> readRangeFiles(final FDate from, final FDate to, final Lock readLock, final ISkipFileFunction skipFileFunction) {
    return new ICloseableIterable<MemoryFileSummary>() {

        @Override
        public ICloseableIterator<MemoryFileSummary> iterator() {
            final FDate usedFrom;
            if (from == null) {
                final V firstValue = getFirstValue();
                if (firstValue == null) {
                    return EmptyCloseableIterator.getInstance();
                }
                usedFrom = extractEndTime.apply(firstValue);
            } else {
                usedFrom = from;
            }
            return new ACloseableIterator<MemoryFileSummary>(new TextDescription("%s[%s]: readRangeFiles(%s, %s)", TimeSeriesStorageCache.class.getSimpleName(), hashKey, from, to)) {

                // use latest time available even if delegate iterator has no values
                private RangeTableRow<String, FDate, MemoryFileSummary> latestFirstTime = fileLookupTable_latestRangeKeyCache.get(usedFrom);

                private final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> delegate;

                {
                    if (latestFirstTime == null) {
                        delegate = EmptyCloseableIterator.getInstance();
                    } else {
                        delegate = getRangeKeys(hashKey, latestFirstTime.getRangeKey().addMilliseconds(1), to);
                    }
                }

                @Override
                protected boolean innerHasNext() {
                    return latestFirstTime != null || delegate.hasNext();
                }

                private ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> getRangeKeys(final String hashKey, final FDate from, final FDate to) {
                    readLock.lock();
                    try {
                        final ICloseableIterator<RangeTableRow<String, FDate, MemoryFileSummary>> range = getAllRangeKeys(readLock).iterator();
                        final GetRangeKeysIterator rangeFiltered = new GetRangeKeysIterator(range, from, to);
                        if (skipFileFunction != null) {
                            return new ASkippingIterator<RangeTableRow<String, FDate, MemoryFileSummary>>(rangeFiltered) {

                                @Override
                                protected boolean skip(final RangeTableRow<String, FDate, MemoryFileSummary> element) {
                                    if (!rangeFiltered.hasNext()) {
                                        /*
                                             * cannot optimize this further for multiple segments because we don't know
                                             * if a segment further back might be empty or not and thus the last segment
                                             * of interest might have been the previous one from which we skipped the
                                             * last file falsely
                                             */
                                        return false;
                                    }
                                    return skipFileFunction.skipFile(element.getValue());
                                }
                            };
                        } else {
                            return rangeFiltered;
                        }
                    } finally {
                        readLock.unlock();
                    }
                }

                @Override
                protected MemoryFileSummary innerNext() {
                    final MemoryFileSummary summary;
                    if (latestFirstTime != null) {
                        summary = latestFirstTime.getValue();
                        latestFirstTime = null;
                    } else {
                        summary = delegate.next().getValue();
                    }
                    return summary;
                }

                @Override
                protected void innerClose() {
                    delegate.close();
                }
            };
        }
    };
}

Example 3 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method isEmptyOrInconsistent.

public boolean isEmptyOrInconsistent() {
    try {
        getFirstValue();
        getLastValue();
    } catch (final Throwable t) {
        if (Throwables.isCausedByType(t, SerializationException.class)) {
            // e.g. fst: unable to find class for code 88 after version upgrade
            log.warn("Table data for [%s] is inconsistent and needs to be reset. Exception during getLastValue: %s", hashKey, t.toString());
            return true;
        } else {
            // unexpected exception, since RemoteFastSerializingSerde only throws SerializingException
            throw Throwables.propagate(t);
        }
    }
    try (ICloseableIterator<MemoryFileSummary> summaries = readRangeFiles(null, null, DisabledLock.INSTANCE, null).iterator()) {
        boolean noFileFound = true;
        while (summaries.hasNext()) {
            final MemoryFileSummary summary = summaries.next();
            final File memoryFile = new File(summary.getMemoryResourceUri());
            final long memoryFileLength = memoryFile.length();
            final long segmentLength = summary.getMemoryOffset() + summary.getMemoryLength();
            if (segmentLength > memoryFileLength) {
                log.warn("Table data for [%s] is inconsistent and needs to be reset. Empty file: [%s]", hashKey, summary);
                return true;
            }
            noFileFound = false;
        }
        return noFileFound;
    }
}

Example 4 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method finishFile.

public void finishFile(final FDate time, final V firstValue, final V lastValue, final int valueCount, final String memoryResourceUri, final long memoryOffset, final long memoryLength) {
    storage.getFileLookupTable().put(hashKey, time, new MemoryFileSummary(valueSerde, firstValue, lastValue, valueCount, memoryResourceUri, memoryOffset, memoryLength));
    clearCaches();
}

Example 5 with MemoryFileSummary

use of de.invesdwin.context.persistence.timeseriesdb.storage.MemoryFileSummary in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method getNextValue.

public V getNextValue(final FDate date, final int shiftForwardUnits) {
    assertShiftUnitsPositiveNonZero(shiftForwardUnits);
    final V lastValue = getLastValue();
    if (lastValue == null) {
        return null;
    }
    final FDate lastTime = extractEndTime.apply(lastValue);
    if (date.isAfterOrEqualTo(lastTime)) {
        return lastValue;
    } else {
        final SingleValue value = storage.getOrLoad_nextValueLookupTable(hashKey, date, shiftForwardUnits, () -> {
            final MutableReference<V> nextValue = new MutableReference<>();
            final MutableInt shiftForwardRemaining = new MutableInt(shiftForwardUnits);
            try (ICloseableIterator<V> rangeValues = readRangeValues(date, null, DisabledLock.INSTANCE, new ISkipFileFunction() {

                @Override
                public boolean skipFile(final MemoryFileSummary file) {
                    final boolean skip = nextValue.get() != null && file.getValueCount() < shiftForwardRemaining.intValue();
                    if (skip) {
                        shiftForwardRemaining.subtract(file.getValueCount());
                    }
                    return skip;
                }
            })) {
                while (shiftForwardRemaining.intValue() >= 0) {
                    nextValue.set(rangeValues.next());
                    shiftForwardRemaining.decrement();
                }
            } catch (final NoSuchElementException e) {
            // ignore
            }
            return new SingleValue(valueSerde, nextValue.get());
        });
        return value.getValue(valueSerde);
    }
}