Examples with FDate de.invesdwin.util.time.fdate.FDate used on opensource projects

Example 11 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class ChannelPerformanceTest method read.

private void read(final ISynchronousWriter requestWriter, final ISynchronousReader responseReader) {
    final Instant readsStart = new Instant();
    FDate prevValue = null;
    int count = 0;
    try {
        if (DEBUG) {
            log.info("client open request writer");
        }
        requestWriter.open();
        if (DEBUG) {
            log.info("client open response reader");
        }
        responseReader.open();
        final ASpinWait spinWait = new ASpinWait() {

            @Override
            protected boolean isConditionFulfilled() throws IOException {
                return responseReader.hasNext();
            }
        };
        Instant waitingSince = new Instant();
        while (true) {
            requestWriter.write(MESSAGE_TYPE, Bytes.EMPTY_ARRAY);
            if (DEBUG) {
                log.info("client request out");
            }
            Assertions.checkTrue(spinWait.awaitFulfill(waitingSince, MAX_WAIT_DURATION));
            final Pair<Integer, byte[]> readMessage = responseReader.readMessage();
            if (DEBUG) {
                log.info("client response in");
            }
            final int messageType = readMessage.getFirst();
            final byte[] responseBytes = readMessage.getSecond();
            Assertions.checkEquals(messageType, MESSAGE_TYPE);
            Assertions.checkEquals(responseBytes.length, MESSAGE_SIZE);
            final FDate value = FDateSerde.GET.fromBytes(responseBytes);
            if (prevValue != null) {
                Assertions.checkTrue(prevValue.isBefore(value));
            }
            prevValue = value;
            count++;
            waitingSince = new Instant();
        }
    } catch (final EOFException e) {
    // writer closed
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
    Assertions.checkEquals(count, VALUES);
    try {
        if (DEBUG) {
            log.info("client close response reader");
        }
        responseReader.close();
        if (DEBUG) {
            log.info("client close request writer");
        }
        requestWriter.close();
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
    printProgress("ReadsFinished", readsStart, VALUES, VALUES);
}

Example 12 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimeSeriesStorageCache method readRangeValues.

protected ICloseableIterator<V> readRangeValues(final FDate from, final FDate to) {
    final ICloseableIterator<File> fileIterator = readRangeFiles(from, to).iterator();
    final ICloseableIterator<ICloseableIterator<V>> chunkIterator = new ATransformingCloseableIterator<File, ICloseableIterator<V>>(fileIterator) {

        private boolean first = true;

        @Override
        protected ICloseableIterator<V> transform(final File value) {
            final ICloseableIterable<V> serializingCollection = newSerializingCollection(value);
            if (first) {
                first = false;
                if (hasNext()) {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            return time.isBefore(from);
                        }
                    };
                // first and last
                } else {
                    return new ASkippingIterator<V>(serializingCollection.iterator()) {

                        @Override
                        protected boolean skip(final V element) {
                            final FDate time = extractTime.apply(element);
                            if (time.isBefore(from)) {
                                return true;
                            } else if (time.isAfter(to)) {
                                throw new FastNoSuchElementException("getRangeValues reached end");
                            }
                            return false;
                        }
                    };
                }
            // last
            } else if (!hasNext()) {
                return new ASkippingIterator<V>(serializingCollection.iterator()) {

                    @Override
                    protected boolean skip(final V element) {
                        final FDate time = extractTime.apply(element);
                        if (time.isAfter(to)) {
                            throw new FastNoSuchElementException("getRangeValues reached end");
                        }
                        return false;
                    }
                };
            } else {
                return serializingCollection.iterator();
            }
        }
    };
    // final ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>> transformer = new ATransformingCloseableIterator<ICloseableIterator<V>, ICloseableIterator<V>>(
    // chunkIterator) {
    // @Override
    // protected ICloseableIterator<V> transform(final ICloseableIterator<V> value) {
    // //keep file open as shortly as possible to fix too many open files exception
    // return new BufferingIterator<V>(value);
    // }
    // };
    // single threaded is 20% better than with producerqueue
    final FlatteningIterator<V> flatteningIterator = new FlatteningIterator<V>(chunkIterator);
    return flatteningIterator;
}

Example 13 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class DatabasePerformanceTest method testTimeSeriesDbPerformance.

@Test
public void testTimeSeriesDbPerformance() throws IncompleteUpdateFoundException {
    final ATimeSeriesDB<String, FDate> table = new ATimeSeriesDB<String, FDate>("testTimeSeriesDbPerformance") {

        @Override
        protected Serde<FDate> newValueSerde() {
            return FDateSerde.GET;
        }

        @Override
        protected Integer newFixedLength() {
            return FDateSerde.FIXED_LENGTH;
        }

        @Override
        protected String hashKeyToString(final String key) {
            return "testTimeSeriesDbPerformance_" + key;
        }

        @Override
        protected FDate extractTime(final FDate value) {
            return value;
        }
    };
    final Instant writesStart = new Instant();
    final ATimeSeriesUpdater<String, FDate> updater = new ATimeSeriesUpdater<String, FDate>(HASH_KEY, table) {

        @Override
        protected ICloseableIterable<FDate> getSource(final FDate updateFrom) {
            return newValues();
        }

        @Override
        protected void onUpdateFinished(final Instant updateStart) {
            printProgress("WritesFinished", writesStart, VALUES, VALUES);
        }

        @Override
        protected void onUpdateStart() {
        }

        @Override
        protected FDate extractTime(final FDate element) {
            return element;
        }

        @Override
        protected FDate extractEndTime(final FDate element) {
            return element;
        }

        @Override
        protected void onFlush(final int flushIndex, final Instant flushStart, final ATimeSeriesUpdater<String, FDate>.UpdateProgress updateProgress) {
            printProgress("Writes", writesStart, updateProgress.getCount() * flushIndex, VALUES);
        }
    };
    Assertions.checkTrue(updater.update());
    final Instant readsStart = new Instant();
    for (int reads = 1; reads <= READS; reads++) {
        FDate prevValue = null;
        final ICloseableIterator<? extends FDate> range = table.rangeValues(HASH_KEY, null, null);
        int count = 0;
        while (true) {
            try {
                final FDate value = range.next();
                if (prevValue != null) {
                    Assertions.checkTrue(prevValue.isBefore(value));
                }
                prevValue = value;
                count++;
            } catch (final NoSuchElementException e) {
                break;
            }
        }
        Assertions.checkEquals(count, VALUES);
        printProgress("Reads", readsStart, VALUES * reads, VALUES * READS);
    }
    printProgress("ReadsFinished", readsStart, VALUES * READS, VALUES * READS);
}

Example 14 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class TimedDecimalSerde method fromBytes.

@Override
public TimedDecimal fromBytes(final byte[] bytes) {
    final ByteBuffer buffer = ByteBuffer.wrap(bytes);
    final FDate time = FDates.extractFDate(buffer);
    final Decimal percent = Decimal.extractDecimal(buffer);
    final TimedDecimal timedMoney = new TimedDecimal(time, percent);
    return timedMoney;
}

Example 15 with FDate

use of de.invesdwin.util.time.fdate.FDate in project invesdwin-context-persistence by subes.

the class ADataUpdater method doUpdate.

protected final FDate doUpdate() throws IncompleteUpdateFoundException {
    final ATimeSeriesUpdater<K, V> updater = new ATimeSeriesUpdater<K, V>(key, getTable()) {

        private static final int BATCH_LOG_INTERVAL = 100_000 / BATCH_FLUSH_INTERVAL;

        @GuardedBy("this")
        private Integer lastFlushIndex;

        @GuardedBy("this")
        private Duration flushDuration;

        @GuardedBy("this")
        private long flushElementCount;

        @GuardedBy("this")
        private FDate lastFlushMaxTime;

        @GuardedBy("this")
        private Instant lastFlushTime;

        @Override
        protected FDate extractTime(final V element) {
            return ADataUpdater.this.extractTime(element);
        }

        @Override
        protected FDate extractEndTime(final V element) {
            return ADataUpdater.this.extractEndTime(element);
        }

        @Override
        protected ICloseableIterable<? extends V> getSource(final FDate updateFrom) {
            final ICloseableIterable<? extends V> downloadElements = downloadElements(key, updateFrom);
            return downloadElements;
        }

        @Override
        protected void onUpdateStart() {
            log.info("Updating %s for [%s]", getElementsName(), keyToString(key));
        }

        @Override
        protected synchronized void onFlush(final int flushIndex, final Instant flushStart, final UpdateProgress progress) {
            lastFlushIndex = Integers.max(lastFlushIndex, flushIndex);
            if (flushDuration == null) {
                flushDuration = flushStart.toDuration();
            } else {
                flushDuration = flushDuration.add(flushStart.toDuration());
            }
            flushElementCount += progress.getCount();
            lastFlushMaxTime = FDates.max(lastFlushMaxTime, progress.getMaxTime());
            if (flushIndex % BATCH_LOG_INTERVAL == 0) {
                logFlush();
            }
        }

        private void logFlush() {
            Assertions.assertThat(lastFlushIndex).isNotNull();
            // if we are too fast, only print status once a second
            if (lastFlushTime == null || lastFlushTime.toDuration().isGreaterThan(Duration.ONE_SECOND)) {
                log.info("Persisted %s. %s batch for [%s]. Reached time [%s]. Processed [%s] during %s", lastFlushIndex, getElementsName(), keyToString(key), lastFlushMaxTime, new ProcessedEventsRateString(flushElementCount, flushDuration), flushDuration);
                lastFlushTime = new Instant();
            }
            lastFlushIndex = null;
            flushDuration = null;
            flushElementCount = 0;
            lastFlushMaxTime = null;
        }

        @Override
        protected synchronized void onUpdateFinished(final Instant updateStart) {
            if (lastFlushIndex != null) {
                logFlush();
            }
            log.info("Finished updating %s %s for [%s] from [%s] to [%s] after %s", getCount(), getElementsName(), keyToString(key), getMinTime(), getMaxTime(), updateStart);
        }
    };
    updater.update();
    return updater.getMaxTime();
}