Examples with TableWriter io.questdb.cairo.TableWriter used on opensource projects

Example 51 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class AlterTableAlterSymbolColumnCacheFlagTest method testAlterSymbolCacheFlagToFalseAndCheckOpenReaderWithCursor.

@Test
public void testAlterSymbolCacheFlagToFalseAndCheckOpenReaderWithCursor() throws Exception {
    String expectedOrdered = "sym\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "googl\n" + "ibm\n" + "ibm\n" + "msft\n" + "msft\n";
    String expectedChronological = "sym\n" + "msft\n" + "googl\n" + "googl\n" + "ibm\n" + "googl\n" + "ibm\n" + "googl\n" + "googl\n" + "googl\n" + "msft\n";
    final RecordCursorPrinter printer = new SingleColumnRecordCursorPrinter(1);
    assertMemoryLeak(() -> {
        assertMemoryLeak(this::createX);
        assertQueryPlain(expectedOrdered, "select sym from x order by sym");
        try (TableReader reader = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "x")) {
            // check cursor before altering symbol column
            sink.clear();
            printer.print(reader.getCursor(), reader.getMetadata(), true, sink);
            Assert.assertEquals(expectedChronological, sink.toString());
            try (TableWriter writer = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "x", "testing")) {
                writer.changeCacheFlag(1, false);
            }
            // reload reader
            Assert.assertTrue(reader.reload());
            // check cursor after reload
            sink.clear();
            printer.print(reader.getCursor(), reader.getMetadata(), true, sink);
            Assert.assertEquals(expectedChronological, sink.toString());
            try (TableReader reader2 = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "x")) {
                sink.clear();
                printer.print(reader2.getCursor(), reader2.getMetadata(), true, sink);
                Assert.assertEquals(expectedChronological, sink.toString());
            }
        }
    });
    assertQueryPlain(expectedOrdered, "select sym from x order by 1 asc");
}

Example 52 with TableWriter

use of io.questdb.cairo.TableWriter in project invesdwin-context-persistence by subes.

the class QuestDBPerformanceTest method testQuestDbPerformance.

@Test
public void testQuestDbPerformance() throws InterruptedException, SqlException, IOException {
    final File directory = new File(ContextProperties.getCacheDirectory(), QuestDBPerformanceTest.class.getSimpleName());
    Files.deleteNative(directory);
    Files.forceMkdir(directory);
    final CairoConfiguration configuration = new DefaultCairoConfiguration(directory.getAbsolutePath());
    final Instant writesStart = new Instant();
    int i = 0;
    final CairoEngine engine = new CairoEngine(configuration);
    final SqlExecutionContextImpl ctx = new SqlExecutionContextImpl(engine, 1);
    final LoopInterruptedCheck loopCheck = new LoopInterruptedCheck(Duration.ONE_SECOND);
    try (SqlCompiler compiler = new SqlCompiler(engine)) {
        compiler.compile("create table abc (value long, key timestamp) timestamp(key)", ctx);
        try (TableWriter writer = engine.getWriter(ctx.getCairoSecurityContext(), "abc", "insert")) {
            for (final FDate date : newValues()) {
                final TableWriter.Row row = writer.newRow(date.millisValue());
                row.putLong(0, date.millisValue());
                row.append();
                i++;
                if (i % FLUSH_INTERVAL == 0) {
                    if (loopCheck.check()) {
                        printProgress("Writes", writesStart, i, VALUES);
                    }
                    writer.commit();
                }
            }
            writer.commit();
        }
        printProgress("WritesFinished", writesStart, VALUES, VALUES);
    }
    readIterator(engine);
    readGet(engine);
    readGetLatest(engine);
    try (SqlCompiler compiler = new SqlCompiler(engine)) {
        compiler.compile("dtop table 'abc';", ctx);
    }
    engine.close();
    Files.deleteNative(directory);
}

Example 53 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class PGJobContextTest method testCairoException.

@Test
public void testCairoException() throws Exception {
    assertMemoryLeak(() -> {
        try (final PGWireServer ignored = createPGServer(2);
            final Connection connection = getConnection(false, true)) {
            connection.prepareStatement("create table xyz(a int)").execute();
            try (TableWriter ignored1 = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "xyz", "testing")) {
                connection.prepareStatement("drop table xyz").execute();
                Assert.fail();
            } catch (SQLException e) {
                TestUtils.assertContains(e.getMessage(), "Could not lock 'xyz'");
                Assert.assertEquals("00000", e.getSQLState());
            }
        }
    });
}

Example 54 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method assertInsertWaitsWhenWriterLocked.

private void assertInsertWaitsWhenWriterLocked() throws Exception {
    final int parallelCount = 2;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(parallelCount).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run(engine -> {
        // create table
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
        TableWriter writer = lockWriter(engine, "balances_x");
        final int insertCount = 10;
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        for (int i = 0; i < parallelCount; i++) {
            new Thread(() -> {
                try {
                    for (int r = 0; r < insertCount; r++) {
                        // insert one record
                        try {
                            new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0Ainsert+into+balances_x+(cust_id%2C+balance_ccy%2C+balance%2C+timestamp)+values+(1%2C+%27USD%27%2C+1500.00%2C+6000000001)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
                        } catch (Exception e) {
                            LOG.error().$("Failed execute insert http request. Server error ").$(e).$();
                        }
                    }
                } finally {
                    countDownLatch.countDown();
                }
            }).start();
        }
        boolean finished = countDownLatch.await(200, TimeUnit.MILLISECONDS);
        // Cairo engine should not allow second writer to be opened on the same table
        // Cairo is expected to have finished == false
        Assert.assertFalse(finished);
        writer.close();
        countDownLatch.await();
        // check if we have parallelCount x insertCount  records
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=select+count(*)+from+balances_x&count=true HTTP/1.1\r\n", "71\r\n" + "{\"query\":\"select count(*) from balances_x\",\"columns\":[{\"name\":\"count\",\"type\":\"LONG\"}],\"dataset\":[[" + parallelCount * insertCount + "]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
    });
}

Example 55 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method assertInsertWaitsExceedsRerunProcessingQueueSize.

private void assertInsertWaitsExceedsRerunProcessingQueueSize() throws Exception {
    final int rerunProcessingQueueSize = 1;
    final int parallelCount = 4;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(parallelCount).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder().withRerunProcessingQueueSize(rerunProcessingQueueSize)).withTelemetry(false).run(engine -> {
        // create table
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
        TableWriter writer = lockWriter(engine, "balances_x");
        final int insertCount = rerunProcessingQueueSize * 10;
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        AtomicInteger fails = new AtomicInteger();
        for (int i = 0; i < parallelCount; i++) {
            new Thread(() -> {
                try {
                    for (int r = 0; r < insertCount; r++) {
                        // insert one record
                        try {
                            new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0Ainsert+into+balances_x+(cust_id%2C+balance_ccy%2C+balance%2C+timestamp)+values+(1%2C+%27USD%27%2C+1500.00%2C+6000000001)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
                        } catch (AssertionError ase) {
                            fails.incrementAndGet();
                        } catch (Exception e) {
                            LOG.error().$("Failed execute insert http request. Server error ").$(e);
                        }
                    }
                } finally {
                    countDownLatch.countDown();
                }
            }).start();
        }
        boolean finished = countDownLatch.await(200, TimeUnit.MILLISECONDS);
        Assert.assertFalse(finished);
        writer.close();
        if (!countDownLatch.await(5000, TimeUnit.MILLISECONDS)) {
            Assert.fail("Wait to process retries exceeded timeout");
        }
        // check if we have parallelCount x insertCount  records
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=select+count(*)+from+balances_x&count=true HTTP/1.1\r\n", "71\r\n" + "{\"query\":\"select count(*) from balances_x\",\"columns\":[{\"name\":\"count\",\"type\":\"LONG\"}],\"dataset\":[[" + (parallelCount * insertCount - fails.get()) + "]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
    });
}