Search in sources :

Example 1 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class DatabasePerformanceTest method testLevelDbPerformance.

@Test
public void testLevelDbPerformance() {
    final ADelegateRangeTable<String, FDate, FDate> table = new ADelegateRangeTable<String, FDate, FDate>("testLevelDbPerformance") {

        @Override
        protected File getDirectory() {
            return new File(ContextProperties.getCacheDirectory(), ADelegateRangeTable.class.getSimpleName());
        }

        @Override
        protected Serde<FDate> newValueSerde() {
            return FDateSerde.GET;
        }

        @Override
        protected Serde<FDate> newRangeKeySerde() {
            return FDateSerde.GET;
        }
    };
    RangeBatch<String, FDate, FDate> batch = table.newRangeBatch();
    final Instant writesStart = new Instant();
    int i = 0;
    for (final FDate date : newValues()) {
        batch.put(HASH_KEY, date, date);
        i++;
        if (i % FLUSH_INTERVAL == 0) {
            printProgress("Writes", writesStart, i, VALUES);
            try {
                batch.flush();
                batch.close();
            } catch (final IOException e) {
                throw new RuntimeException(e);
            }
            batch = table.newRangeBatch();
        }
    }
    try {
        batch.flush();
        batch.close();
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
    printProgress("WritesFinished", writesStart, VALUES, VALUES);
    final Instant readsStart = new Instant();
    for (int reads = 1; reads <= READS; reads++) {
        FDate prevValue = null;
        final ICloseableIterator<FDate> range = table.rangeValues(HASH_KEY);
        int count = 0;
        while (true) {
            try {
                final FDate value = range.next();
                if (prevValue != null) {
                    Assertions.checkTrue(prevValue.isBefore(value));
                }
                prevValue = value;
                count++;
            } catch (final NoSuchElementException e) {
                break;
            }
        }
        Assertions.checkEquals(count, VALUES);
        printProgress("Reads", readsStart, VALUES * reads, VALUES * READS);
    }
    printProgress("ReadsFinished", readsStart, VALUES * READS, VALUES * READS);
}
Also used : Instant(de.invesdwin.util.time.Instant) ProcessedEventsRateString(de.invesdwin.util.lang.ProcessedEventsRateString) IOException(java.io.IOException) ADelegateRangeTable(de.invesdwin.context.persistence.leveldb.ezdb.ADelegateRangeTable) File(java.io.File) FDate(de.invesdwin.util.time.fdate.FDate) NoSuchElementException(java.util.NoSuchElementException) ATest(de.invesdwin.context.test.ATest) Test(org.junit.Test)

Example 2 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class TestStockData method assertIteration.

private void assertIteration(final int countFDates, final FDate fromFDate, final FDate toFDate) {
    TableIterator<String, FDate, Integer> range = table.range(MSFT, fromFDate, toFDate);
    int iteratedBars = 0;
    int prevValue = 0;
    FDate left1000FDate = null;
    FDate left900FDate = null;
    final Instant start = new Instant();
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        final Integer value = next.getValue();
        // System.out.println(value);
        iteratedBars++;
        Assertions.checkTrue(prevValue < value);
        prevValue = value;
        if (iteratedBars == countFDates - 999) {
            left1000FDate = next.getRangeKey();
        }
        if (iteratedBars == countFDates - 900) {
            left900FDate = next.getRangeKey();
        }
    }
    System.out.println("took: " + start);
    Assertions.checkEquals(countFDates, iteratedBars);
    Assertions.checkEquals(1, table.getLatest(MSFT, fromFDate).getValue());
    Assertions.checkEquals(countFDates, table.getLatest(MSFT, toFDate).getValue());
    // System.out.println(left1000FDate +" -> "+left900FDate);
    range = table.range(MSFT, left1000FDate, left900FDate);
    int curLeftIt = 0;
    TableRow<String, FDate, Integer> prev = null;
    while (range.hasNext()) {
        final TableRow<String, FDate, Integer> next = range.next();
        curLeftIt++;
        Assertions.checkEquals(countFDates - 1000 + curLeftIt, next.getValue());
        if (prev != null) {
            final Integer nextFromPrevPlus = table.getNext(MSFT, new FDate(prev.getRangeKey().millisValue() + 1)).getValue();
            Assertions.checkEquals(next.getValue(), nextFromPrevPlus);
            final Integer prevFromNextMinus = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue() - 1)).getValue();
            Assertions.checkEquals(prev.getValue(), prevFromNextMinus);
        }
        final Integer nextFromNextIsSame = table.getNext(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), nextFromNextIsSame);
        final Integer prevFromNextIsSame = table.getPrev(MSFT, new FDate(next.getRangeKey().millisValue())).getValue();
        Assertions.checkEquals(next.getValue(), prevFromNextIsSame);
        prev = next;
    }
    Assertions.checkEquals(100, curLeftIt);
}
Also used : Instant(de.invesdwin.util.time.Instant) FDate(de.invesdwin.util.time.fdate.FDate)

Example 3 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class ASpinWaitTest method testMaxWait.

// CHECKSTYLE:ON
@Test
public void testMaxWait() throws IOException {
    final ASpinWait waitingSpinWait = new ASpinWait() {

        @Override
        protected boolean isConditionFulfilled() {
            return false;
        }
    };
    final Duration maxWait = Duration.ONE_SECOND;
    final Instant waitingSince = new Instant();
    waitingSpinWait.awaitFulfill(waitingSince, maxWait);
    Assertions.assertThat(waitingSince.toDuration()).isGreaterThanOrEqualTo(maxWait);
}
Also used : Instant(de.invesdwin.util.time.Instant) Duration(de.invesdwin.util.time.duration.Duration) ATest(de.invesdwin.context.test.ATest) Test(org.junit.Test)

Example 4 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class ATimeSeriesUpdater method update.

public final boolean update() throws IncompleteUpdateFoundException {
    try {
        if (!table.getTableLock(key).writeLock().tryLock(1, TimeUnit.MINUTES)) {
            throw new RetryLaterRuntimeException("Write lock could not be acquired for table [" + table.getName() + "] and key [" + key + "]. Please ensure all iterators are closed!");
        }
    } catch (final InterruptedException e1) {
        throw new RuntimeException(e1);
    }
    try {
        if (updateLockFile.exists()) {
            throw new IncompleteUpdateFoundException("Incomplete update found for table [" + table.getName() + "], need to clean everything up to restore all from scratch.");
        }
        try {
            FileUtils.touch(updateLockFile);
        } catch (final IOException e) {
            throw new RuntimeException(e);
        }
        final Instant updateStart = new Instant();
        onUpdateStart();
        doUpdate();
        onUpdateFinished(updateStart);
        Assertions.assertThat(updateLockFile.delete()).isTrue();
        return true;
    } finally {
        table.getTableLock(key).writeLock().unlock();
    }
}
Also used : RetryLaterRuntimeException(de.invesdwin.context.integration.retry.RetryLaterRuntimeException) Instant(de.invesdwin.util.time.Instant) RetryLaterRuntimeException(de.invesdwin.context.integration.retry.RetryLaterRuntimeException) IOException(java.io.IOException)

Example 5 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class InfluxDBPerformanceTest method testInfluxDbPerformanceAsync.

@Test
public void testInfluxDbPerformanceAsync() throws Exception {
    final int freeHttpPort = Network.getFreeServerPort();
    final int freeUdpPort = Network.getFreeServerPort();
    final InfluxServer server = startInfluxDB(freeHttpPort, freeUdpPort);
    server.start();
    try {
        Thread.sleep(10 * 1000);
        final String dbname = "influxDbPerformance";
        final String policyname = "defaultPolicy";
        final String measurementName = "measurementsPerformance";
        final InfluxDB influxDB = InfluxDBFactory.connect("http://localhost:" + freeHttpPort);
        influxDB.createDatabase(dbname);
        influxDB.createRetentionPolicy(policyname, dbname, "9999d", 1, true);
        BatchPoints batch = BatchPoints.database(dbname).retentionPolicy(policyname).build();
        final Instant writesStart = new Instant();
        int i = 0;
        for (final FDate date : newValues()) {
            final Point point = Point.measurement(measurementName).time(date.millisValue(), TimeUnit.MILLISECONDS).tag("hashKey", HASH_KEY).addField("value", date.millisValue()).build();
            batch.point(point);
            i++;
            if (i % FLUSH_INTERVAL == 0) {
                printProgress("Writes", writesStart, i, VALUES);
                influxDB.write(batch);
                batch = BatchPoints.database(dbname).retentionPolicy(policyname).build();
            }
        }
        influxDB.write(batch);
        batch = null;
        printProgress("WritesFinished", writesStart, VALUES, VALUES);
        TimeUnit.SECONDS.sleep(1);
        final Instant readsStart = new Instant();
        for (int reads = 1; reads <= READS; reads++) {
            final IMutableReference<FDate> prevValueRef = new VolatileReference<>();
            final AtomicBoolean finished = new AtomicBoolean();
            final int readsFinal = reads;
            final AtomicInteger count = new AtomicInteger();
            influxDB.query(new Query("Select value from " + measurementName + " where hashKey = '" + HASH_KEY + "'", dbname), 10_000, new Consumer<QueryResult>() {

                @Override
                public void accept(final QueryResult queryResult) {
                    try {
                        final List<Result> range = queryResult.getResults();
                        if (range == null || range.isEmpty()) {
                            return;
                        }
                        final Result firstResult = range.get(0);
                        if (firstResult == null) {
                            return;
                        }
                        final List<Series> series = firstResult.getSeries();
                        if (series == null || series.isEmpty()) {
                            return;
                        }
                        final List<List<Object>> firstSeries = series.get(0).getValues();
                        if (firstSeries == null) {
                            return;
                        }
                        for (int result = 0; result < firstSeries.size(); result++) {
                            final Double valueDouble = (Double) firstSeries.get(result).get(1);
                            final FDate value = new FDate(valueDouble.longValue());
                            final FDate prevValue = prevValueRef.get();
                            if (prevValue != null) {
                                Assertions.checkTrue(prevValue.isBefore(value));
                            }
                            prevValueRef.set(value);
                            count.incrementAndGet();
                        }
                    } catch (final Throwable t) {
                        Err.process(t);
                    }
                }
            }, () -> finished.set(true));
            while (!finished.get() && count.get() != VALUES) {
                TimeUnit.NANOSECONDS.sleep(1);
            }
            Assertions.checkEquals(count.get(), VALUES);
            printProgress("Reads", readsStart, VALUES * readsFinal, VALUES * READS);
        }
        printProgress("ReadsFinished", readsStart, VALUES * READS, VALUES * READS);
    } finally {
        server.stop();
    }
}
Also used : BatchPoints(org.influxdb.dto.BatchPoints) Query(org.influxdb.dto.Query) Instant(de.invesdwin.util.time.Instant) VolatileReference(de.invesdwin.util.concurrent.reference.VolatileReference) ProcessedEventsRateString(de.invesdwin.util.lang.ProcessedEventsRateString) Point(org.influxdb.dto.Point) Point(org.influxdb.dto.Point) FDate(de.invesdwin.util.time.date.FDate) QueryResult(org.influxdb.dto.QueryResult) Result(org.influxdb.dto.QueryResult.Result) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) QueryResult(org.influxdb.dto.QueryResult) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) InfluxDB(org.influxdb.InfluxDB) InfluxServer(io.apisense.embed.influx.InfluxServer) List(java.util.List) ATest(de.invesdwin.context.test.ATest) Test(org.junit.jupiter.api.Test)

Aggregations

Instant (de.invesdwin.util.time.Instant)93 FDate (de.invesdwin.util.time.date.FDate)78 LoopInterruptedCheck (de.invesdwin.util.concurrent.loop.LoopInterruptedCheck)65 NoSuchElementException (java.util.NoSuchElementException)30 Test (org.junit.jupiter.api.Test)25 File (java.io.File)16 PreparedStatement (java.sql.PreparedStatement)16 ResultSet (java.sql.ResultSet)15 Statement (java.sql.Statement)12 ATest (de.invesdwin.context.test.ATest)8 RecordFile (com.indeed.lsmtree.recordlog.RecordFile)6 ProcessedEventsRateString (de.invesdwin.util.lang.ProcessedEventsRateString)6 TimedDecimal (de.invesdwin.util.math.decimal.TimedDecimal)5 FDate (de.invesdwin.util.time.fdate.FDate)5 IOException (java.io.IOException)5 Connection (java.sql.Connection)5 APersistentMap (de.invesdwin.context.integration.persistentmap.APersistentMap)4 ATimeSeriesUpdater (de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater)4 Test (org.junit.Test)4 SnappyCodec (com.indeed.util.compress.SnappyCodec)3