Examples with FDate de.invesdwin.util.time.fdate.FDate used on opensource projects

Example 6 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class DatabasePerformanceTest method testLevelDbPerformance.

@Test
public void testLevelDbPerformance() {
    final ADelegateRangeTable<String, FDate, FDate> table = new ADelegateRangeTable<String, FDate, FDate>("testLevelDbPerformance") {

        @Override
        protected File getDirectory() {
            return new File(ContextProperties.getCacheDirectory(), ADelegateRangeTable.class.getSimpleName());
        }

        @Override
        protected Serde<FDate> newValueSerde() {
            return FDateSerde.GET;
        }

        @Override
        protected Serde<FDate> newRangeKeySerde() {
            return FDateSerde.GET;
        }
    };
    RangeBatch<String, FDate, FDate> batch = table.newRangeBatch();
    final Instant writesStart = new Instant();
    int i = 0;
    for (final FDate date : newValues()) {
        batch.put(HASH_KEY, date, date);
        i++;
        if (i % FLUSH_INTERVAL == 0) {
            printProgress("Writes", writesStart, i, VALUES);
            try {
                batch.flush();
                batch.close();
            } catch (final IOException e) {
                throw new RuntimeException(e);
            }
            batch = table.newRangeBatch();
        }
    }
    try {
        batch.flush();
        batch.close();
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
    printProgress("WritesFinished", writesStart, VALUES, VALUES);
    final Instant readsStart = new Instant();
    for (int reads = 1; reads <= READS; reads++) {
        FDate prevValue = null;
        final ICloseableIterator<FDate> range = table.rangeValues(HASH_KEY);
        int count = 0;
        while (true) {
            try {
                final FDate value = range.next();
                if (prevValue != null) {
                    Assertions.checkTrue(prevValue.isBefore(value));
                }
                prevValue = value;
                count++;
            } catch (final NoSuchElementException e) {
                break;
            }
        }
        Assertions.checkEquals(count, VALUES);
        printProgress("Reads", readsStart, VALUES * reads, VALUES * READS);
    }
    printProgress("ReadsFinished", readsStart, VALUES * READS, VALUES * READS);
}

Example 7 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TestStockData method assertIteration.

private void assertIteration(final int countFDates, final FDate fromFDate, final FDate toFDate) {
    TableIterator<String, FDate, Integer> range = table.range(MSFT, fromFDate, toFDate);
    int iteratedBars = 0;
    int prevValue = 0;
    FDate left1000FDate = null;
    FDate left900FDate = null;
    final Instant start = new Instant();
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        final Integer value = next.getValue();
        // System.out.println(value);
        iteratedBars++;
        Assertions.checkTrue(prevValue < value);
        prevValue = value;
        if (iteratedBars == countFDates - 999) {
            left1000FDate = next.getRangeKey();
        }
        if (iteratedBars == countFDates - 900) {
            left900FDate = next.getRangeKey();
        }
    }
    System.out.println("took: " + start);
    Assertions.checkEquals(countFDates, iteratedBars);
    Assertions.checkEquals(1, table.getLatest(MSFT, fromFDate).getValue());
    Assertions.checkEquals(countFDates, table.getLatest(MSFT, toFDate).getValue());
    // System.out.println(left1000FDate +" -> "+left900FDate);
    range = table.range(MSFT, left1000FDate, left900FDate);
    int curLeftIt = 0;
    TableRow<String, FDate, Integer> prev = null;
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        curLeftIt++;
        Assertions.checkEquals(countFDates - 1000 + curLeftIt, next.getValue());
        if (prev != null) {
            final Integer nextFromPrevPlus = table.getNext(MSFT, new FDate(prev.getRangeKey().millisValue() + 1)).getValue();
            Assertions.checkEquals(next.getValue(), nextFromPrevPlus);
            final Integer prevFromNextMinus = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue() - 1)).getValue();
            Assertions.checkEquals(prev.getValue(), prevFromNextMinus);
        }
        final Integer nextFromNextIsSame = table.getNext(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), nextFromNextIsSame);
        final Integer prevFromNextIsSame = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), prevFromNextIsSame);
        prev = next;
    }
    Assertions.checkEquals(100, curLeftIt);
}

Example 8 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class ATimeSeriesUpdater method doUpdate.

private void doUpdate() {
    final Pair<FDate, List<V>> pair = lookupTable.prepareForUpdate();
    final FDate updateFrom = pair.getFirst();
    final List<V> lastValues = pair.getSecond();
    Assertions.checkNotNull(lastValues);
    ICloseableIterable<? extends V> source = getSource(updateFrom);
    if (updateFrom != null) {
        // ensure we add no duplicate values
        source = new ASkippingIterable<V>(source) {

            @Override
            protected boolean skip(final V element) {
                return extractTime(element).isBefore(updateFrom);
            }
        };
    }
    final FlatteningIterable<? extends V> flatteningSources = new FlatteningIterable<>(lastValues, source);
    try (ICloseableIterator<? extends V> elements = flatteningSources.iterator()) {
        final ICloseableIterator<UpdateProgress> batchWriterProducer = new ICloseableIterator<UpdateProgress>() {

            @Override
            public boolean hasNext() {
                return elements.hasNext();
            }

            @Override
            public UpdateProgress next() {
                final UpdateProgress progress = new UpdateProgress();
                while (elements.hasNext()) {
                    final V element = elements.next();
                    if (progress.onElement(element)) {
                        return progress;
                    }
                }
                return progress;
            }

            @Override
            public void close() {
                elements.close();
            }
        };
        // do IO in a different thread than batch filling
        try (ACloseableIterator<UpdateProgress> batchProducer = new AProducerQueueIterator<UpdateProgress>(getClass().getSimpleName() + "_batchProducer_" + table.hashKeyToString(key), BATCH_QUEUE_SIZE) {

            @Override
            protected ICloseableIterator<ATimeSeriesUpdater<K, V>.UpdateProgress> newProducer() {
                return batchWriterProducer;
            }
        }) {
            final AtomicInteger flushIndex = new AtomicInteger();
            try (ACloseableIterator<UpdateProgress> parallelConsumer = new AParallelChunkConsumerIterator<UpdateProgress, UpdateProgress>(getClass().getSimpleName() + "_batchConsumer_" + table.hashKeyToString(key), batchProducer, BATCH_WRITER_THREADS) {

                @Override
                protected UpdateProgress doWork(final UpdateProgress request) {
                    request.write(flushIndex.incrementAndGet());
                    return request;
                }
            }) {
                while (parallelConsumer.hasNext()) {
                    final UpdateProgress progress = parallelConsumer.next();
                    count += progress.getCount();
                    if (minTime == null) {
                        minTime = progress.getMinTime();
                    }
                    maxTime = progress.getMaxTime();
                }
            }
        }
    }
}

Example 9 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class LevelDBTest method testInverseOrder.

@Test
public void testInverseOrder() {
    final ADelegateRangeTable<String, FDate, Integer> rangeTable = new ADelegateRangeTable<String, FDate, Integer>("testInverseOrder") {

        @Override
        protected Serde<FDate> newRangeKeySerde() {
            return FDateSerde.GET;
        }

        @Override
        protected boolean allowHasNext() {
            return true;
        }

        @Override
        protected boolean allowPutWithoutBatch() {
            return true;
        }
    };
    final FDate now = FDateBuilder.newDate(2000);
    final FDate oneDate = now.addDays(1);
    final FDate twoDate = now.addDays(2);
    final FDate threeDate = now.addDays(3);
    rangeTable.put(HASHKEY, oneDate, 1);
    rangeTable.put(HASHKEY, twoDate, 2);
    rangeTable.put(HASHKEY, threeDate, 3);
    final DelegateTableIterator<String, FDate, Integer> range3 = rangeTable.range(HASHKEY, now);
    Assertions.assertThat(range3.next().getValue()).isEqualTo(1);
    Assertions.assertThat(range3.next().getValue()).isEqualTo(2);
    Assertions.assertThat(range3.next().getValue()).isEqualTo(3);
    Assertions.assertThat(range3.hasNext()).isFalse();
    try {
        range3.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    // should already be closed but should not cause an error when calling again
    range3.close();
    final DelegateTableIterator<String, FDate, Integer> rangeNone = rangeTable.range(HASHKEY);
    Assertions.assertThat(rangeNone.next().getValue()).isEqualTo(1);
    Assertions.assertThat(rangeNone.next().getValue()).isEqualTo(2);
    Assertions.assertThat(rangeNone.next().getValue()).isEqualTo(3);
    Assertions.assertThat(rangeNone.hasNext()).isFalse();
    try {
        rangeNone.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final DelegateTableIterator<String, FDate, Integer> rangeMin = rangeTable.range(HASHKEY, FDate.MIN_DATE);
    Assertions.assertThat(rangeMin.next().getValue()).isEqualTo(1);
    Assertions.assertThat(rangeMin.next().getValue()).isEqualTo(2);
    Assertions.assertThat(rangeMin.next().getValue()).isEqualTo(3);
    Assertions.assertThat(rangeMin.hasNext()).isFalse();
    try {
        rangeMin.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final DelegateTableIterator<String, FDate, Integer> rangeMax = rangeTable.range(HASHKEY, FDate.MAX_DATE);
    Assertions.assertThat(rangeMax.hasNext()).isFalse();
    try {
        rangeMax.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    final DelegateTableIterator<String, FDate, Integer> range2 = rangeTable.range(HASHKEY, twoDate);
    Assertions.assertThat(range2.next().getValue()).isEqualTo(2);
    Assertions.assertThat(range2.next().getValue()).isEqualTo(3);
    Assertions.assertThat(range2.hasNext()).isFalse();
    try {
        range2.next();
        Fail.fail("Exception expected!");
    } catch (final NoSuchElementException e) {
        Assertions.assertThat(e).isNotNull();
    }
    testReverse(rangeTable, oneDate, twoDate, threeDate);
    testGetLatestForRange(rangeTable, oneDate, twoDate, threeDate);
    rangeTable.deleteTable();
}

Example 10 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TestStockData method testStockData.

@Test
public void testStockData() throws IOException, ParseException {
    final InputStream in = new ClassPathResource("MSFT.txt", getClass()).getInputStream();
    final List<String> lines = CharStreams.readLines(new InputStreamReader(in));
    lines.remove(0);
    lines.remove(0);
    Collections.reverse(lines);
    in.close();
    Long prevLongTime = null;
    int countFDate = 0;
    FDate firstFDate = null;
    FDate lastFDate = null;
    for (final String line : lines) {
        final String[] split = line.split(",");
        Assertions.checkEquals(7, split.length);
        countFDate++;
        final String dateStr = split[0];
        final FDate date = FDate.valueOf(dateStr, "yyyy-MM-dd");
        if (firstFDate == null) {
            firstFDate = date;
        }
        lastFDate = date;
        final long longTime = date.millisValue();
        if (prevLongTime != null) {
            // System.out.println(dateStr + ":"+date + " - "+prevLongTime+"  < " + longTime + " -> "
            // + (prevLongTime < longTime));
            Assertions.checkTrue(prevLongTime < longTime);
        }
        table.put(MSFT, date, countFDate);
        prevLongTime = longTime;
    }
    System.out.println(MSFT + " has " + countFDate + " bars");
    assertIteration(countFDate, MIN_DATE, MAX_DATE);
    assertIteration(countFDate, firstFDate, lastFDate);
    // Fri Jan 24 23:46:40 UTC 2014
    TableIterator<String, FDate, Integer> range = table.range(MSFT, FDateBuilder.newDate(2014, 1, 23));
    int countBars = 0;
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(253, countBars);
    range = table.range(MSFT, FDateBuilder.newDate(2014, 1, 23), null);
    countBars = 0;
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        // System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(253, countBars);
    range = table.range(MSFT, null, FDateBuilder.newDate(1987, 1, 1));
    countBars = 0;
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        // System.out.println(next.getValue());
        countBars++;
    }
    Assertions.checkEquals(204, countBars);
}