Examples with TableWriter io.questdb.cairo.TableWriter used on opensource projects

Example 36 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method testImportRerunsExceedsRerunProcessingQueueSize.

public void testImportRerunsExceedsRerunProcessingQueueSize(int startDelay) throws Exception {
    final int rerunProcessingQueueSize = 1;
    final int parallelCount = 4;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(2).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder().withNetwork(getSendDelayNetworkFacade(startDelay)).withRerunProcessingQueueSize(rerunProcessingQueueSize)).run(engine -> {
        // create table and do 1 import
        new SendAndReceiveRequestBuilder().execute(ValidImportRequest, ValidImportResponse);
        TableWriter writer = lockWriter(engine, "fhv_tripdata_2017-02.csv");
        final int validRequestRecordCount = 24;
        final int insertCount = 4;
        AtomicInteger failedImports = new AtomicInteger();
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        for (int i = 0; i < parallelCount; i++) {
            int finalI = i;
            new Thread(() -> {
                try {
                    for (int r = 0; r < insertCount; r++) {
                        // insert one record
                        try {
                            new SendAndReceiveRequestBuilder().execute(ValidImportRequest, ValidImportResponse);
                        } catch (AssertionError e) {
                            LOG.info().$("Server call succeeded but response is different from the expected one").$();
                            failedImports.incrementAndGet();
                        } catch (Exception e) {
                            LOG.error().$("Failed execute insert http request. Server error ").$(e).$();
                        }
                    }
                } finally {
                    countDownLatch.countDown();
                }
                LOG.info().$("Stopped thread ").$(finalI).$();
            }).start();
        }
        boolean finished = countDownLatch.await(100, TimeUnit.MILLISECONDS);
        Assert.assertFalse(finished);
        writer.close();
        if (!countDownLatch.await(5000, TimeUnit.MILLISECONDS)) {
            Assert.fail("Imports did not finish within reasonable time");
        }
        // check if we have parallelCount x insertCount  records
        LOG.info().$("Requesting row count").$();
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=select+count(*)+from+%22fhv_tripdata_2017-02.csv%22&count=true HTTP/1.1\r\n", "84\r\n" + "{\"query\":\"select count(*) from \\\"fhv_tripdata_2017-02.csv\\\"\",\"columns\":[{\"name\":\"count\",\"type\":\"LONG\"}],\"dataset\":[[" + (parallelCount * insertCount + 1 - failedImports.get()) * validRequestRecordCount + "]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
    });
}

Example 37 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method testRenameWaitsWhenWriterLocked.

@Test
public void testRenameWaitsWhenWriterLocked() throws Exception {
    final int parallelCount = 2;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(parallelCount).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run(engine -> {
        // create table
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
        TableWriter writer = lockWriter(engine, "balances_x");
        CountDownLatch countDownLatch = new CountDownLatch(1);
        new Thread(() -> {
            try {
                try {
                    // Rename table
                    new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=rename+table+%27balances_x%27+to+%27balances_y%27&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
                } catch (Exception e) {
                    LOG.error().$("Failed execute insert http request. Server error ").$(e).$();
                }
            } finally {
                countDownLatch.countDown();
            }
        }).start();
        boolean finished = countDownLatch.await(200, TimeUnit.MILLISECONDS);
        // Cairo engine should not allow table rename while writer is opened
        // Cairo is expected to have finished == false
        Assert.assertFalse(finished);
        writer.close();
        Assert.assertTrue("Table rename did not complete within timeout after writer is released", countDownLatch.await(5, TimeUnit.SECONDS));
    });
}

Example 38 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class MinCharGroupByFunctionFactoryTest method testAllNull.

@Test
public void testAllNull() throws SqlException {
    compiler.compile("create table tab (f char)", sqlExecutionContext);
    try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "tab", "testing")) {
        for (int i = 100; i > 10; i--) {
            TableWriter.Row r = w.newRow();
            r.append();
        }
        w.commit();
    }
    try (RecordCursorFactory factory = compiler.compile("select min(f) from tab", sqlExecutionContext).getRecordCursorFactory()) {
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            Record record = cursor.getRecord();
            Assert.assertEquals(1, cursor.size());
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(0, record.getChar(0));
        }
    }
}

Example 39 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class MinDoubleGroupByFunctionFactoryTest method testNonNull.

@Test
public void testNonNull() throws SqlException {
    compiler.compile("create table tab (f double)", sqlExecutionContext);
    final Rnd rnd = new Rnd();
    try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "tab", "testing")) {
        for (int i = 100; i > 10; i--) {
            TableWriter.Row r = w.newRow();
            r.putDouble(0, rnd.nextDouble());
            r.append();
        }
        w.commit();
    }
    try (RecordCursorFactory factory = compiler.compile("select min(f) from tab", sqlExecutionContext).getRecordCursorFactory()) {
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            Record record = cursor.getRecord();
            Assert.assertEquals(1, cursor.size());
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(0.0011075139045715332, record.getDouble(0), 0.0001);
        }
    }
}

Example 40 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class MinDoubleGroupByFunctionFactoryTest method testFirstNull.

@Test
public void testFirstNull() throws SqlException {
    compiler.compile("create table tab (f double)", sqlExecutionContext);
    final Rnd rnd = new Rnd();
    try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "tab", "testing")) {
        TableWriter.Row r = w.newRow();
        r.append();
        for (int i = 100; i > 10; i--) {
            r = w.newRow();
            r.putDouble(0, rnd.nextDouble());
            r.append();
        }
        w.commit();
    }
    try (RecordCursorFactory factory = compiler.compile("select min(f) from tab", sqlExecutionContext).getRecordCursorFactory()) {
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            Record record = cursor.getRecord();
            Assert.assertEquals(1, cursor.size());
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(0.0011075139045715332, record.getDouble(0), 0.0001);
        }
    }
}