Search in sources :

Example 11 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class RecordLogDirectoryPerformanceTest method testRecordLogDirectoryPerformance.

@Test
public void testRecordLogDirectoryPerformance() throws IOException, InterruptedException {
    final File directory = new File(ContextProperties.getCacheDirectory(), RecordLogDirectoryPerformanceTest.class.getSimpleName());
    Files.deleteNative(directory);
    Files.forceMkdir(directory);
    final File file = new File(directory, "testRecordLogDirectoryPerformance");
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    final Instant writesStart = new Instant();
    try (RecordLogDirectory.Writer<TimedDecimal> writer = RecordLogDirectory.Writer.create(file, IndeedSerializer.valueOf(TimedDecimalSerde.GET), new SnappyCodec(), Integer.MAX_VALUE)) {
        int i = 0;
        for (final FDate date : newValues()) {
            writer.append(new TimedDecimal(date, date.millisValue()));
            i++;
            if (i % FLUSH_INTERVAL == 0) {
                if (loopCheck.check()) {
                    printProgress("Writes", writesStart, i, VALUES);
                }
            // writer.roll();
            }
        }
        writer.roll();
        printProgress("WritesFinished", writesStart, VALUES, VALUES);
    }
    readIterator(file);
}
Also used : Instant(de.invesdwin.util.time.Instant) RecordLogDirectory(com.indeed.lsmtree.recordlog.RecordLogDirectory) LoopInterruptedCheck(de.invesdwin.util.concurrent.loop.LoopInterruptedCheck) File(java.io.File) RecordFile(com.indeed.lsmtree.recordlog.RecordFile) TimedDecimal(de.invesdwin.util.math.decimal.TimedDecimal) SnappyCodec(com.indeed.util.compress.SnappyCodec) FDate(de.invesdwin.util.time.date.FDate) Test(org.junit.jupiter.api.Test)

Example 12 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class RecordLogDirectoryPerformanceTest method readIterator.

private void readIterator(final File file) throws IOException, InterruptedException {
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck();
    final Instant readsStart = new Instant();
    for (int reads = 1; reads <= READS; reads++) {
        FDate prevValue = null;
        int count = 0;
        try (RecordLogDirectory<TimedDecimal> reader = new RecordLogDirectory.Builder<TimedDecimal>(file, IndeedSerializer.valueOf(TimedDecimalSerde.GET), new SnappyCodec()).build()) {
            final RecordFile.Reader<TimedDecimal> iterator = reader.reader();
            while (iterator.next()) {
                final FDate value = iterator.get().getTime();
                if (prevValue != null) {
                    Assertions.checkTrue(prevValue.isBefore(value));
                }
                prevValue = value;
                count++;
            }
            Assertions.checkEquals(count, VALUES);
            if (loopCheck.check()) {
                printProgress("Reads", readsStart, VALUES * reads, VALUES * READS);
            }
        }
    }
    printProgress("ReadsFinished", readsStart, VALUES * READS, VALUES * READS);
}
Also used : RecordFile(com.indeed.lsmtree.recordlog.RecordFile) Instant(de.invesdwin.util.time.Instant) RecordLogDirectory(com.indeed.lsmtree.recordlog.RecordLogDirectory) LoopInterruptedCheck(de.invesdwin.util.concurrent.loop.LoopInterruptedCheck) TimedDecimal(de.invesdwin.util.math.decimal.TimedDecimal) SnappyCodec(com.indeed.util.compress.SnappyCodec) FDate(de.invesdwin.util.time.date.FDate)

Example 13 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class DuckDBPerformanceTest method testDuckDbPerformance.

@Test
public void testDuckDbPerformance() throws Exception {
    final File directory = new File(ContextProperties.getCacheDirectory(), DuckDBPerformanceTest.class.getSimpleName());
    Files.deleteNative(directory);
    Files.forceMkdirParent(directory);
    final Instant writesStart = new Instant();
    int i = 0;
    Class.forName("org.duckdb.DuckDBDriver");
    final Connection conn = DriverManager.getConnection("jdbc:duckdb:" + directory.getAbsolutePath());
    final Statement create = conn.createStatement();
    create.execute("CREATE TABLE abc (key LONG, value LONG, PRIMARY KEY(key))");
    create.execute("CREATE UNIQUE INDEX idx_abc on abc (key desc)");
    create.close();
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    final Statement insert = conn.createStatement();
    insert.execute("BEGIN TRANSACTION");
    final StringBuilder insertValues = new StringBuilder();
    for (final FDate date : newValues()) {
        final long value = date.longValue(FTimeUnit.MILLISECONDS);
        if (insertValues.length() > 0) {
            insertValues.append(",");
        }
        insertValues.append("(" + value + "," + value + ")");
        i++;
        if (i % FLUSH_INTERVAL == 0) {
            insert.execute("INSERT INTO abc VALUES " + insertValues);
            insertValues.setLength(0);
            if (loopCheck.check()) {
                printProgress("Writes", writesStart, i, VALUES);
            }
        }
    }
    if (insertValues.length() > 0) {
        insert.execute("INSERT INTO abc VALUES " + insertValues);
        insertValues.setLength(0);
    }
    insert.execute("COMMIT");
    printProgress("WritesFinished", writesStart, VALUES, VALUES);
    insert.close();
    readIterator(conn);
    readGet(conn);
    readGetLatest(conn);
    final Statement drop = conn.createStatement();
    drop.execute("drop table abc");
    drop.close();
    Files.deleteNative(directory);
}
Also used : Statement(java.sql.Statement) Instant(de.invesdwin.util.time.Instant) Connection(java.sql.Connection) LoopInterruptedCheck(de.invesdwin.util.concurrent.loop.LoopInterruptedCheck) File(java.io.File) FDate(de.invesdwin.util.time.date.FDate) Test(org.junit.jupiter.api.Test)

Example 14 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class H2PerformanceTest method readGetLatest.

private void readGetLatest(final Connection conn) throws Exception {
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    final List<FDate> values = Lists.toList(newValues());
    final Instant readsStart = new Instant();
    for (int reads = 0; reads < READS; reads++) {
        FDate prevValue = null;
        int count = 0;
        try (PreparedStatement select = conn.prepareStatement("SELECT v FROM abc WHERE k <=? ORDER BY k DESC LIMIT 1")) {
            for (int i = 0; i < values.size(); i++) {
                select.setLong(1, values.get(i).millisValue());
                try (ResultSet results = select.executeQuery()) {
                    Assertions.checkTrue(results.next());
                    final FDate value = new FDate(results.getLong(1));
                    if (prevValue != null) {
                        Assertions.checkTrue(prevValue.isBefore(value));
                    }
                    prevValue = value;
                    count++;
                    if (loopCheck.check()) {
                        printProgress("GetLatests", readsStart, VALUES * reads + count, VALUES * READS);
                    }
                }
            }
        }
        Assertions.checkEquals(count, VALUES);
    }
    printProgress("GetLatestsFinished", readsStart, VALUES * READS, VALUES * READS);
}
Also used : Instant(de.invesdwin.util.time.Instant) ResultSet(java.sql.ResultSet) PreparedStatement(java.sql.PreparedStatement) LoopInterruptedCheck(de.invesdwin.util.concurrent.loop.LoopInterruptedCheck) FDate(de.invesdwin.util.time.date.FDate)

Example 15 with Instant

use of de.invesdwin.util.time.Instant in project invesdwin-context-persistence by subes.

the class H2PerformanceTest method testH2Performance.

@Test
public void testH2Performance() throws Exception {
    final File directory = new File(ContextProperties.getCacheDirectory(), H2PerformanceTest.class.getSimpleName());
    Files.deleteNative(directory);
    Files.forceMkdirParent(directory);
    final Instant writesStart = new Instant();
    int i = 0;
    Class.forName("org.h2.Driver");
    final Connection conn = DriverManager.getConnection("jdbc:h2:" + directory.getAbsolutePath(), "sa", "");
    final Statement create = conn.createStatement();
    create.execute("DROP TABLE abc IF EXISTS");
    create.execute("CREATE TABLE abc (k LONG, v LONG, PRIMARY KEY(k))");
    // asc vs desc index order matters a lot
    create.execute("CREATE UNIQUE INDEX idx_abc on abc (k desc)");
    create.close();
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    final Statement tx = conn.createStatement();
    tx.execute("BEGIN TRANSACTION");
    final PreparedStatement insert = conn.prepareStatement("INSERT INTO abc VALUES (?,?)");
    for (final FDate date : newValues()) {
        final long value = date.longValue(FTimeUnit.MILLISECONDS);
        insert.setLong(1, value);
        insert.setLong(2, value);
        insert.addBatch();
        i++;
        if (i % FLUSH_INTERVAL == 0) {
            insert.executeBatch();
            tx.execute("COMMIT");
            tx.execute("BEGIN TRANSACTION");
            if (loopCheck.check()) {
                printProgress("Writes", writesStart, i, VALUES);
            }
        }
    }
    insert.executeBatch();
    insert.close();
    tx.execute("COMMIT");
    printProgress("WritesFinished", writesStart, VALUES, VALUES);
    tx.close();
    readIterator(conn);
    readGet(conn);
    readGetLatest(conn);
    final Statement drop = conn.createStatement();
    drop.execute("drop table abc");
    drop.close();
    Files.deleteNative(directory);
}
Also used : PreparedStatement(java.sql.PreparedStatement) Statement(java.sql.Statement) Instant(de.invesdwin.util.time.Instant) Connection(java.sql.Connection) PreparedStatement(java.sql.PreparedStatement) LoopInterruptedCheck(de.invesdwin.util.concurrent.loop.LoopInterruptedCheck) File(java.io.File) FDate(de.invesdwin.util.time.date.FDate) Test(org.junit.jupiter.api.Test)

Aggregations

Instant (de.invesdwin.util.time.Instant)93 FDate (de.invesdwin.util.time.date.FDate)78 LoopInterruptedCheck (de.invesdwin.util.concurrent.loop.LoopInterruptedCheck)65 NoSuchElementException (java.util.NoSuchElementException)30 Test (org.junit.jupiter.api.Test)25 File (java.io.File)16 PreparedStatement (java.sql.PreparedStatement)16 ResultSet (java.sql.ResultSet)15 Statement (java.sql.Statement)12 ATest (de.invesdwin.context.test.ATest)8 RecordFile (com.indeed.lsmtree.recordlog.RecordFile)6 ProcessedEventsRateString (de.invesdwin.util.lang.ProcessedEventsRateString)6 TimedDecimal (de.invesdwin.util.math.decimal.TimedDecimal)5 FDate (de.invesdwin.util.time.fdate.FDate)5 IOException (java.io.IOException)5 Connection (java.sql.Connection)5 APersistentMap (de.invesdwin.context.integration.persistentmap.APersistentMap)4 ATimeSeriesUpdater (de.invesdwin.context.persistence.timeseriesdb.updater.ATimeSeriesUpdater)4 Test (org.junit.Test)4 SnappyCodec (com.indeed.util.compress.SnappyCodec)3