Examples with ATimeSeriesUpdater de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater used on opensource projects

Example 1 with ATimeSeriesUpdater

use of de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater in project invesdwin-context-persistence by subes.

the class ATimeSeriesDBTest method testGetPreviousAndNext.

// CHECKSTYLE:ON
@Test
public void testGetPreviousAndNext() throws IncompleteUpdateFoundException {
    final String key = "asdf";
    final ATimeSeriesDB<String, FDate> table = new ATimeSeriesDB<String, FDate>("testGetPrevious") {

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }
    };
    final List<FDate> dates = new ArrayList<>();
    for (int i = 2000; i <= 2010; i++) {
        dates.add(FDateBuilder.newDate(i));
    }
    new ATimeSeriesUpdater<String, FDate>(key, table) {

        @Override
        protected ICloseableIterable<? extends FDate> getSource(final FDate updateFrom) {
            return WrapperCloseableIterable.maybeWrap(dates);
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final FDate element) {
            return element;
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<String, FDate>.UpdateProgress updateProgress) {
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    }.update();
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getPreviousValue(key, dates.get(dates.size() - 1), i);
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getPreviousValue(key, FDate.MAX_DATE, i);
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getPreviousValue(key, FDate.MIN_DATE, i);
        final FDate expectedValue = dates.get(0);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getNextValue(key, dates.get(0), i);
        final FDate expectedValue = dates.get(i);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getNextValue(key, FDate.MIN_DATE, i);
        final FDate expectedValue = dates.get(i);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
    for (int i = 1; i < dates.size(); i++) {
        final FDate value = table.getNextValue(key, FDate.MAX_DATE, i);
        final FDate expectedValue = dates.get(dates.size() - 1);
        Assertions.checkEquals(value, expectedValue, i + ": expected [" + expectedValue + "] got [" + value + "]");
    }
}

Example 2 with ATimeSeriesUpdater

use of de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater in project invesdwin-context-persistence by subes.

the class ATimeSeriesDBTest method testGetPreviousAndNextSkipFile.

@Test
public void testGetPreviousAndNextSkipFile() throws IncompleteUpdateFoundException {
    final String key = "asdf";
    final ATimeSeriesDB<String, FDate> table = new ATimeSeriesDB<String, FDate>("testGetPreviousSkipFile") {

        @Override
        protected ISerde<FDate> newValueSerde() {
            return new TypeDelegateSerde<FDate>(FDate.class);
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return key;
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }
    };
    final List<FDate> dates = new ArrayList<>();
    for (int i = 0; i < 100_000; i++) {
        dates.add(new FDate(i));
    }
    final MutableInt segments = new MutableInt();
    new ATimeSeriesUpdater<String, FDate>(key, table) {

        @Override
        protected ICloseableIterable<? extends FDate> getSource(final FDate updateFrom) {
            return WrapperCloseableIterable.maybeWrap(dates);
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final FDate element) {
            return element;
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<String, FDate>.UpdateProgress updateProgress) {
            segments.increment();
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    }.update();
    Assertions.assertThat(segments.intValue()).isEqualByComparingTo(10);
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, dates.get(dates.size() - 1), i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - i - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, FDate.MAX_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(0);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getPreviousValue(key, FDate.MIN_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(i);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, dates.get(0), i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(i);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, FDate.MIN_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
    for (int i = 0; i < dates.size(); i += ATimeSeriesUpdater.BATCH_FLUSH_INTERVAL) {
        final FDate expectedValue = dates.get(dates.size() - 1);
        final long expectedIndex = expectedValue.millisValue();
        final FDate value = table.getNextValue(key, FDate.MAX_DATE, i);
        final long valueIndex = value.millisValue();
        Assertions.checkEquals(valueIndex, expectedIndex, i + ": expected [" + expectedIndex + "] got [" + valueIndex + "]");
    }
}

Example 3 with ATimeSeriesUpdater

use of de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater in project invesdwin-context-persistence by subes.

the class TimeseriesDBPerformanceTest method testTimeSeriesDbPerformance.

@Test
public void testTimeSeriesDbPerformance() throws IncompleteUpdateFoundException, InterruptedException {
    final ATimeSeriesDB<String, FDate> table = new ATimeSeriesDB<String, FDate>("testTimeSeriesDbPerformance") {

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.TEMP_DIRECTORY;
        }

        @Override
        protected ISerde<FDate> newValueSerde() {
            return FDateSerde.GET;
        }

        @Override
        protected Integer newValueFixedLength() {
            return FDateSerde.FIXED_LENGTH;
        }

        @Override
        protected String innerHashKeyToString(final String key) {
            return "testTimeSeriesDbPerformance_" + key;
        }

        @Override
        protected FDate extractEndTime(final FDate value) {
            return value;
        }
    };
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    final Instant writesStart = new Instant();
    final ATimeSeriesUpdater<String, FDate> updater = new ATimeSeriesUpdater<String, FDate>(HASH_KEY, table) {

        @Override
        protected ICloseableIterable<FDate> getSource(final FDate updateFrom) {
            return newValues();
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
            printProgress("WritesFinished", writesStart, VALUES, VALUES);
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final FDate element) {
            return element;
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<String, FDate>.UpdateProgress updateProgress) {
            try {
                if (loopCheck.check()) {
                    printProgress("Writes", writesStart, updateProgress.getValueCount() * flushIndex, VALUES);
                }
            } catch (final InterruptedException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    };
    Assertions.checkTrue(updater.update());
    readIterator(table, "Cold", 1);
    readIterator(table, "Warm", READS);
    readGetLatest(table, "Cold", 1);
    readGetLatest(table, "Warm", READS);
}

Example 4 with ATimeSeriesUpdater

use of de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater in project invesdwin-context-persistence by subes.

the class PersistentLiveSegment method putNextLiveValues.

public void putNextLiveValues(final ICloseableIterable<V> memoryValues) {
    final ADelegateRangeTable<String, TimeRange, SegmentStatus> segmentStatusTable = historicalSegmentTable.getStorage().getSegmentStatusTable();
    final SegmentStatus existingStatus = segmentStatusTable.get(hashKey, segmentedKey.getSegment());
    if (existingStatus == null) {
        segmentStatusTable.put(hashKey, segmentedKey.getSegment(), SegmentStatus.INITIALIZING);
    } else if (existingStatus != SegmentStatus.INITIALIZING) {
        throw UnknownArgumentException.newInstance(SegmentStatus.class, existingStatus);
    }
    final ATimeSeriesUpdater<SegmentedKey<K>, V> updater = new ATimeSeriesUpdater<SegmentedKey<K>, V>(segmentedKey, table) {

        @Override
        protected ICloseableIterable<? extends V> getSource(final FDate updateFrom) {
            return memoryValues;
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final V element) {
            return historicalSegmentTable.extractEndTime(element);
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<SegmentedKey<K>, V>.UpdateProgress updateProgress) {
        }

        @Override
        protected boolean shouldRedoLastFile() {
            return false;
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    };
    try {
        Assertions.checkTrue(updater.update());
    } catch (final IncompleteUpdateFoundException e) {
        throw new RuntimeException(e);
    }
    empty = false;
}