Examples with SegmentedKey de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey used on opensource projects

Example 1 with SegmentedKey

use of de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey in project invesdwin-context-persistence by subes.

the class LiveSegmentedTimeSeriesStorageCache method putNextLiveValue.

public void putNextLiveValue(final V nextLiveValue) {
    final FDate nextLiveKey = historicalSegmentTable.extractEndTime(nextLiveValue);
    final FDate lastAvailableHistoricalSegmentTo = historicalSegmentTable.getLastAvailableHistoricalSegmentTo(key, nextLiveKey);
    final TimeRange segment = historicalSegmentTable.getSegmentFinder(key).query().getValue(nextLiveKey);
    if (lastAvailableHistoricalSegmentTo.isAfterNotNullSafe(segment.getFrom()) && /*
                 * allow equals since on first value of the next bar we might get an overlap for once when the last
                 * available time was updated beforehand
                 */
    !lastAvailableHistoricalSegmentTo.equalsNotNullSafe(segment.getTo())) {
        throw new IllegalStateException("lastAvailableHistoricalSegmentTo [" + lastAvailableHistoricalSegmentTo + "] should be before or equal to liveSegmentFrom [" + segment.getFrom() + "]");
    }
    if (liveSegment != null && nextLiveKey.isAfter(liveSegment.getSegmentedKey().getSegment().getTo())) {
        if (!lastAvailableHistoricalSegmentTo.isBeforeOrEqualTo(liveSegment.getSegmentedKey().getSegment().getTo())) {
            throw new IllegalStateException("lastAvailableHistoricalSegmentTo [" + lastAvailableHistoricalSegmentTo + "] should be before or equal to liveSegmentTo [" + segment.getTo() + "]");
        }
        liveSegment.convertLiveSegmentToHistorical();
        liveSegment.close();
        liveSegment = null;
    }
    if (liveSegment == null) {
        final SegmentedKey<K> segmentedKey = new SegmentedKey<K>(key, segment);
        liveSegment = new SwitchingLiveSegment<K, V>(segmentedKey, historicalSegmentTable, batchFlushInterval);
    }
    liveSegment.putNextLiveValue(nextLiveKey, nextLiveValue);
}

Example 2 with SegmentedKey

use of de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey in project invesdwin-context-persistence by subes.

the class FileLiveSegmentTest method testInverseOrder.

@Test
public void testInverseOrder() {
    final Map<Integer, FDate> extractTime = new HashMap<>();
    final SegmentedKey<FDate> segmentedKey = new SegmentedKey<FDate>(FDate.MIN_DATE, new TimeRange(FDate.MIN_DATE, FDate.MAX_DATE));
    final ALiveSegmentedTimeSeriesDB<FDate, Integer> timeSeriesDB = new ALiveSegmentedTimeSeriesDB<FDate, Integer>("testInverseOrder") {

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.getCacheDirectory();
        }

        @Override
        protected ICloseableIterable<? extends Integer> downloadSegmentElements(final SegmentedKey<FDate> segmentedKey) {
            throw new UnsupportedOperationException();
        }

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected ISerde<Integer> newValueSerde() {
            return IntegerSerde.GET;
        }

        @Override
        protected FDate extractEndTime(final Integer value) {
            return extractTime.get(value);
        }

        @Override
        protected String innerHashKeyToString(final FDate key) {
            return key.toString(FDate.FORMAT_UNDERSCORE_DATE_TIME_MS);
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final FDate key, final FDate updateTo) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected String getElementsName() {
            throw new UnsupportedOperationException();
        }
    };
    @SuppressWarnings("unchecked") final ALiveSegmentedTimeSeriesDB<FDate, Integer>.HistoricalSegmentTable historicalSegmentTable = Reflections.field("historicalSegmentTable").ofType(HistoricalSegmentTable.class).in(timeSeriesDB).get();
    final FileLiveSegment<FDate, Integer> rangeTable = new FileLiveSegment<FDate, Integer>(segmentedKey, historicalSegmentTable);
    final FDate now = FDateBuilder.newDate(2000);
    final FDate oneDate = now.addDays(1);
    final FDate twoDate = now.addDays(2);
    final FDate threeDate = now.addDays(3);
    rangeTable.putNextLiveValue(oneDate, 1);
    extractTime.put(1, oneDate);
    rangeTable.putNextLiveValue(twoDate, 2);
    extractTime.put(2, twoDate);
    rangeTable.putNextLiveValue(threeDate, 3);
    extractTime.put(3, threeDate);
    final ICloseableIterator<Integer> range3 = rangeTable.rangeValues(now, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range3.next()).isEqualTo(1);
    Assertions.assertThat(range3.next()).isEqualTo(2);
    Assertions.assertThat(range3.next()).isEqualTo(3);
    Assertions.assertThat(range3.hasNext()).isFalse();
    try {
        range3.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    // should already be closed but should not cause an error when calling again
    range3.close();
    final ICloseableIterator<Integer> rangeNone = rangeTable.rangeValues(null, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeNone.next()).isEqualTo(1);
    Assertions.assertThat(rangeNone.next()).isEqualTo(2);
    Assertions.assertThat(rangeNone.next()).isEqualTo(3);
    Assertions.assertThat(rangeNone.hasNext()).isFalse();
    try {
        rangeNone.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMin = rangeTable.rangeValues(FDate.MIN_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMin.next()).isEqualTo(1);
    Assertions.assertThat(rangeMin.next()).isEqualTo(2);
    Assertions.assertThat(rangeMin.next()).isEqualTo(3);
    Assertions.assertThat(rangeMin.hasNext()).isFalse();
    try {
        rangeMin.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMax = rangeTable.rangeValues(FDate.MAX_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMax.hasNext()).isFalse();
    try {
        rangeMax.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> range2 = rangeTable.rangeValues(twoDate, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range2.next()).isEqualTo(2);
    Assertions.assertThat(range2.next()).isEqualTo(3);
    Assertions.assertThat(range2.hasNext()).isFalse();
    try {
        range2.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    testReverse(rangeTable, oneDate, twoDate, threeDate);
    testGetLatestForRange(rangeTable, oneDate, twoDate, threeDate);
    rangeTable.close();
}

Example 3 with SegmentedKey

use of de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey in project invesdwin-context-persistence by subes.

the class RangeTableLiveSegmentTest method testInverseOrder.

@Test
public void testInverseOrder() {
    final Map<Integer, FDate> extractTime = new HashMap<>();
    final SegmentedKey<FDate> segmentedKey = new SegmentedKey<FDate>(FDate.MIN_DATE, new TimeRange(FDate.MIN_DATE, FDate.MAX_DATE));
    final ALiveSegmentedTimeSeriesDB<FDate, Integer> timeSeriesDB = new ALiveSegmentedTimeSeriesDB<FDate, Integer>("testInverseOrder") {

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.getCacheDirectory();
        }

        @Override
        protected ICloseableIterable<? extends Integer> downloadSegmentElements(final SegmentedKey<FDate> segmentedKey) {
            throw new UnsupportedOperationException();
        }

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected ISerde<Integer> newValueSerde() {
            return IntegerSerde.GET;
        }

        @Override
        protected FDate extractEndTime(final Integer value) {
            return extractTime.get(value);
        }

        @Override
        protected String innerHashKeyToString(final FDate key) {
            return key.toString(FDate.FORMAT_UNDERSCORE_DATE_TIME_MS);
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final FDate key, final FDate updateTo) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected String getElementsName() {
            throw new UnsupportedOperationException();
        }
    };
    @SuppressWarnings("unchecked") final ALiveSegmentedTimeSeriesDB<FDate, Integer>.HistoricalSegmentTable historicalSegmentTable = Reflections.field("historicalSegmentTable").ofType(HistoricalSegmentTable.class).in(timeSeriesDB).get();
    final RangeTableLiveSegment<FDate, Integer> rangeTable = new RangeTableLiveSegment<FDate, Integer>(segmentedKey, historicalSegmentTable);
    final FDate now = FDateBuilder.newDate(2000);
    final FDate oneDate = now.addDays(1);
    final FDate twoDate = now.addDays(2);
    final FDate threeDate = now.addDays(3);
    rangeTable.putNextLiveValue(oneDate, 1);
    extractTime.put(1, oneDate);
    rangeTable.putNextLiveValue(twoDate, 2);
    extractTime.put(2, twoDate);
    rangeTable.putNextLiveValue(threeDate, 3);
    extractTime.put(3, threeDate);
    final ICloseableIterator<Integer> range3 = rangeTable.rangeValues(now, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range3.next()).isEqualTo(1);
    Assertions.assertThat(range3.next()).isEqualTo(2);
    Assertions.assertThat(range3.next()).isEqualTo(3);
    Assertions.assertThat(range3.hasNext()).isFalse();
    try {
        range3.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    // should already be closed but should not cause an error when calling again
    range3.close();
    final ICloseableIterator<Integer> rangeNone = rangeTable.rangeValues(null, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeNone.next()).isEqualTo(1);
    Assertions.assertThat(rangeNone.next()).isEqualTo(2);
    Assertions.assertThat(rangeNone.next()).isEqualTo(3);
    Assertions.assertThat(rangeNone.hasNext()).isFalse();
    try {
        rangeNone.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMin = rangeTable.rangeValues(FDate.MIN_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMin.next()).isEqualTo(1);
    Assertions.assertThat(rangeMin.next()).isEqualTo(2);
    Assertions.assertThat(rangeMin.next()).isEqualTo(3);
    Assertions.assertThat(rangeMin.hasNext()).isFalse();
    try {
        rangeMin.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMax = rangeTable.rangeValues(FDate.MAX_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMax.hasNext()).isFalse();
    try {
        rangeMax.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> range2 = rangeTable.rangeValues(twoDate, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range2.next()).isEqualTo(2);
    Assertions.assertThat(range2.next()).isEqualTo(3);
    Assertions.assertThat(range2.hasNext()).isFalse();
    try {
        range2.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    testReverse(rangeTable, oneDate, twoDate, threeDate);
    testGetLatestForRange(rangeTable, oneDate, twoDate, threeDate);
    rangeTable.close();
}

Example 4 with SegmentedKey

use of de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey in project invesdwin-context-persistence by subes.

the class PersistentLiveSegment method putNextLiveValues.

public void putNextLiveValues(final ICloseableIterable<V> memoryValues) {
    final ADelegateRangeTable<String, TimeRange, SegmentStatus> segmentStatusTable = historicalSegmentTable.getStorage().getSegmentStatusTable();
    final SegmentStatus existingStatus = segmentStatusTable.get(hashKey, segmentedKey.getSegment());
    if (existingStatus == null) {
        segmentStatusTable.put(hashKey, segmentedKey.getSegment(), SegmentStatus.INITIALIZING);
    } else if (existingStatus != SegmentStatus.INITIALIZING) {
        throw UnknownArgumentException.newInstance(SegmentStatus.class, existingStatus);
    }
    final ATimeSeriesUpdater<SegmentedKey<K>, V> updater = new ATimeSeriesUpdater<SegmentedKey<K>, V>(segmentedKey, table) {

        @Override
        protected ICloseableIterable<? extends V> getSource(final FDate updateFrom) {
            return memoryValues;
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractEndTime(final V element) {
            return historicalSegmentTable.extractEndTime(element);
        }

        @Override
        protected void onFlush(final int flushIndex, final ATimeSeriesUpdater<SegmentedKey<K>, V>.UpdateProgress updateProgress) {
        }

        @Override
        protected boolean shouldRedoLastFile() {
            return false;
        }

        @Override
        public Percent getProgress() {
            return null;
        }
    };
    try {
        Assertions.checkTrue(updater.update());
    } catch (final IncompleteUpdateFoundException e) {
        throw new RuntimeException(e);
    }
    empty = false;
}

Example 5 with SegmentedKey

use of de.invesdwin.context.persistence.timeseriesdb.segmented.SegmentedKey in project invesdwin-context-persistence by subes.

the class HeapLiveSegmentTest method testInverseOrder.

@Test
public void testInverseOrder() {
    final Map<Integer, FDate> extractTime = new HashMap<>();
    final SegmentedKey<FDate> segmentedKey = new SegmentedKey<FDate>(FDate.MIN_DATE, new TimeRange(FDate.MIN_DATE, FDate.MAX_DATE));
    final ALiveSegmentedTimeSeriesDB<FDate, Integer> timeSeriesDB = new ALiveSegmentedTimeSeriesDB<FDate, Integer>("testInverseOrder") {

        @Override
        protected File getBaseDirectory() {
            return ContextProperties.getCacheDirectory();
        }

        @Override
        protected ICloseableIterable<? extends Integer> downloadSegmentElements(final SegmentedKey<FDate> segmentedKey) {
            throw new UnsupportedOperationException();
        }

        @Override
        public AHistoricalCache<TimeRange> getSegmentFinder(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected Integer newValueFixedLength() {
            return null;
        }

        @Override
        protected ISerde<Integer> newValueSerde() {
            return IntegerSerde.GET;
        }

        @Override
        protected FDate extractEndTime(final Integer value) {
            return extractTime.get(value);
        }

        @Override
        protected String innerHashKeyToString(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        public FDate getFirstAvailableHistoricalSegmentFrom(final FDate key) {
            throw new UnsupportedOperationException();
        }

        @Override
        public FDate getLastAvailableHistoricalSegmentTo(final FDate key, final FDate updateTo) {
            throw new UnsupportedOperationException();
        }

        @Override
        protected String getElementsName() {
            throw new UnsupportedOperationException();
        }
    };
    @SuppressWarnings("unchecked") final ALiveSegmentedTimeSeriesDB<FDate, Integer>.HistoricalSegmentTable historicalSegmentTable = Reflections.field("historicalSegmentTable").ofType(HistoricalSegmentTable.class).in(timeSeriesDB).get();
    final HeapLiveSegment<FDate, Integer> rangeTable = new HeapLiveSegment<FDate, Integer>(segmentedKey, historicalSegmentTable);
    final FDate now = FDateBuilder.newDate(2000);
    final FDate oneDate = now.addDays(1);
    final FDate twoDate = now.addDays(2);
    final FDate threeDate = now.addDays(3);
    rangeTable.putNextLiveValue(oneDate, 1);
    extractTime.put(1, oneDate);
    rangeTable.putNextLiveValue(twoDate, 2);
    extractTime.put(2, twoDate);
    rangeTable.putNextLiveValue(threeDate, 3);
    extractTime.put(3, threeDate);
    final ICloseableIterator<Integer> range3 = rangeTable.rangeValues(now, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range3.next()).isEqualTo(1);
    Assertions.assertThat(range3.next()).isEqualTo(2);
    Assertions.assertThat(range3.next()).isEqualTo(3);
    Assertions.assertThat(range3.hasNext()).isFalse();
    try {
        range3.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    // should already be closed but should not cause an error when calling again
    range3.close();
    final ICloseableIterator<Integer> rangeNone = rangeTable.rangeValues(null, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeNone.next()).isEqualTo(1);
    Assertions.assertThat(rangeNone.next()).isEqualTo(2);
    Assertions.assertThat(rangeNone.next()).isEqualTo(3);
    Assertions.assertThat(rangeNone.hasNext()).isFalse();
    try {
        rangeNone.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMin = rangeTable.rangeValues(FDate.MIN_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMin.next()).isEqualTo(1);
    Assertions.assertThat(rangeMin.next()).isEqualTo(2);
    Assertions.assertThat(rangeMin.next()).isEqualTo(3);
    Assertions.assertThat(rangeMin.hasNext()).isFalse();
    try {
        rangeMin.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> rangeMax = rangeTable.rangeValues(FDate.MAX_DATE, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(rangeMax.hasNext()).isFalse();
    try {
        rangeMax.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final ICloseableIterator<Integer> range2 = rangeTable.rangeValues(twoDate, null, DisabledLock.INSTANCE, null).iterator();
    Assertions.assertThat(range2.next()).isEqualTo(2);
    Assertions.assertThat(range2.next()).isEqualTo(3);
    Assertions.assertThat(range2.hasNext()).isFalse();
    try {
        range2.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    testReverse(rangeTable, oneDate, twoDate, threeDate);
    testGetLatestForRange(rangeTable, oneDate, twoDate, threeDate);
    rangeTable.close();
}