Examples with TableWriter io.questdb.cairo.TableWriter used on opensource projects

Example 56 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method assertInsertsIsPerformedWhenWriterLockedAndDisconnected.

private void assertInsertsIsPerformedWhenWriterLockedAndDisconnected() throws Exception {
    final int parallelCount = 4;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(parallelCount).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run(engine -> {
        // create table
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n", IODispatcherTest.JSON_DDL_RESPONSE);
        TableWriter writer = lockWriter(engine, "balances_x");
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        long[] fds = new long[parallelCount];
        Arrays.fill(fds, -1);
        Thread[] threads = new Thread[parallelCount];
        for (int i = 0; i < parallelCount; i++) {
            final int threadI = i;
            threads[i] = new Thread(() -> {
                try {
                    // await nothing
                    try {
                        Os.sleep(threadI * 5);
                        String request = "GET /query?query=%0A%0Ainsert+into+balances_x+(cust_id%2C+balance_ccy%2C+balance%2C+timestamp)+values+(" + threadI + "%2C+%27USD%27%2C+1500.00%2C+6000000001)&limit=0%2C1000&count=true HTTP/1.1\r\n" + SendAndReceiveRequestBuilder.RequestHeaders;
                        long fd = new SendAndReceiveRequestBuilder().connectAndSendRequest(request);
                        fds[threadI] = fd;
                    } catch (Exception e) {
                        LOG.error().$("Failed execute insert http request. Server error ").$(e);
                    }
                } finally {
                    countDownLatch.countDown();
                }
            });
            threads[i].start();
        }
        countDownLatch.await();
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=SELECT+1 HTTP/1.1\r\n", "54\r\n" + "{\"query\":\"SELECT 1\",\"columns\":[{\"name\":\"1\",\"type\":\"INT\"}],\"dataset\":[[1]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
        for (int n = 0; n < fds.length; n++) {
            Assert.assertNotEquals(fds[n], -1);
            NetworkFacadeImpl.INSTANCE.close(fds[n]);
        }
        writer.close();
        // check if we have parallelCount x insertCount  records
        int waitCount = 1000 / 50 * parallelCount;
        for (int i = 0; i < waitCount; i++) {
            try {
                new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=select+count()+from+balances_x&count=true HTTP/1.1\r\n", "6f\r\n" + "{\"query\":\"select count() from balances_x\",\"columns\":[{\"name\":\"count\",\"type\":\"LONG\"}],\"dataset\":[[" + parallelCount + "]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
                return;
            } catch (ComparisonFailure e) {
                if (i < waitCount - 1) {
                    Os.sleep(50);
                } else {
                    throw e;
                }
            }
        }
    });
}

Example 57 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method testImportWaitsWhenWriterLocked.

public void testImportWaitsWhenWriterLocked(HttpQueryTestBuilder httpQueryTestBuilder, int slowServerReceiveNetAfterSending, String importRequest, String importResponse, boolean failOnUnfinished, boolean allowFailures) throws Exception {
    final int parallelCount = httpQueryTestBuilder.getWorkerCount();
    httpQueryTestBuilder.run((engine) -> {
        // create table and do 1 import
        new SendAndReceiveRequestBuilder().execute(ValidImportRequest, ValidImportResponse);
        TableWriter writer = lockWriter(engine, "fhv_tripdata_2017-02.csv");
        final int validRequestRecordCount = 24;
        final int insertCount = 1;
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        AtomicInteger successRequests = new AtomicInteger();
        for (int i = 0; i < parallelCount; i++) {
            int finalI = i;
            new Thread(() -> {
                try {
                    for (int r = 0; r < insertCount; r++) {
                        // insert one record
                        try {
                            SendAndReceiveRequestBuilder sendAndReceiveRequestBuilder = new SendAndReceiveRequestBuilder().withNetworkFacade(getSendDelayNetworkFacade(slowServerReceiveNetAfterSending)).withCompareLength(importResponse.length());
                            sendAndReceiveRequestBuilder.execute(importRequest, importResponse);
                            successRequests.incrementAndGet();
                        } catch (AssertionError e) {
                            if (allowFailures) {
                                LOG.info().$("Failed execute insert http request. Comparison failed").$();
                            } else {
                                LOG.error().$("Failed execute insert http request. Comparison failed").$(e).$();
                            }
                        } catch (Exception e) {
                            LOG.error().$("Failed execute insert http request.").$(e).$();
                        }
                    }
                } finally {
                    countDownLatch.countDown();
                }
                LOG.info().$("Stopped thread ").$(finalI).$();
            }).start();
        }
        boolean finished = countDownLatch.await(100, TimeUnit.MILLISECONDS);
        if (failOnUnfinished) {
            // Cairo engine should not allow second writer to be opened on the same table
            // Cairo is expected to have finished == false
            Assert.assertFalse(finished);
        }
        writer.close();
        if (!countDownLatch.await(50000, TimeUnit.MILLISECONDS)) {
            Assert.fail("Imports did not finish within reasonable time");
        }
        if (!allowFailures) {
            Assert.assertEquals(parallelCount, successRequests.get());
        }
        // check if we have parallelCount x insertCount  records
        LOG.info().$("Requesting row count").$();
        int rowsExpected = (successRequests.get() + 1) * validRequestRecordCount;
        new SendAndReceiveRequestBuilder().executeWithStandardHeaders("GET /query?query=select+count(*)+from+%22fhv_tripdata_2017-02.csv%22&count=true HTTP/1.1\r\n", (rowsExpected < 100 ? "83" : "84") + "\r\n" + "{\"query\":\"select count(*) from \\\"fhv_tripdata_2017-02.csv\\\"\",\"columns\":[{\"name\":\"count\",\"type\":\"LONG\"}],\"dataset\":[[" + rowsExpected + "]],\"count\":1}\r\n" + "00\r\n" + "\r\n");
    });
}

Example 58 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class RetryIODispatcherTest method assertImportProcessedWhenClientDisconnected.

private void assertImportProcessedWhenClientDisconnected() throws Exception {
    final int parallelCount = 2;
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(2).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run((engine) -> {
        // create table and do 1 import
        new SendAndReceiveRequestBuilder().execute(ValidImportRequest, ValidImportResponse);
        TableWriter writer = lockWriter(engine, "fhv_tripdata_2017-02.csv");
        final int validRequestRecordCount = 24;
        final int insertCount = 1;
        CountDownLatch countDownLatch = new CountDownLatch(parallelCount);
        long[] fds = new long[parallelCount * insertCount];
        Arrays.fill(fds, -1);
        for (int i = 0; i < parallelCount; i++) {
            final int threadI = i;
            new Thread(() -> {
                try {
                    for (int r = 0; r < insertCount; r++) {
                        // insert one record
                        try {
                            long fd = new SendAndReceiveRequestBuilder().connectAndSendRequest(ValidImportRequest);
                            fds[threadI * insertCount + r] = fd;
                        } catch (Exception e) {
                            LOG.error().$("Failed execute insert http request. Server error ").$(e).$();
                        }
                    }
                } finally {
                    countDownLatch.countDown();
                }
                LOG.info().$("Stopped thread ").$(threadI).$();
            }).start();
        }
        countDownLatch.await();
        assertNRowsInserted(validRequestRecordCount);
        for (int n = 0; n < fds.length; n++) {
            Assert.assertNotEquals(fds[n], -1);
            NetworkFacadeImpl.INSTANCE.close(fds[n]);
        }
        // Cairo engine should not allow second writer to be opened on the same table, all requests should wait for the writer to be available
        writer.close();
        for (int i = 0; i < 20; i++) {
            try {
                // check if we have parallelCount x insertCount  records
                int nRows = (parallelCount + 1) * validRequestRecordCount;
                assertNRowsInserted(nRows);
                return;
            } catch (ComparisonFailure e) {
                if (i < 9) {
                    Os.sleep(50);
                } else {
                    throw e;
                }
            }
        }
    });
}

Example 59 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class FirstDateGroupByFunctionFactoryTest method testAllNull.

@Test
public void testAllNull() throws SqlException {
    compiler.compile("create table tab (f date)", sqlExecutionContext);
    try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "tab", "testing")) {
        for (int i = 100; i > 10; i--) {
            TableWriter.Row r = w.newRow();
            r.append();
        }
        w.commit();
    }
    try (RecordCursorFactory factory = compiler.compile("select first(f) from tab", sqlExecutionContext).getRecordCursorFactory()) {
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            Record record = cursor.getRecord();
            Assert.assertEquals(1, cursor.size());
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(Numbers.LONG_NaN, record.getLong(0));
        }
    }
}

Example 60 with TableWriter

use of io.questdb.cairo.TableWriter in project questdb by bluestreak01.

the class FirstDateGroupByFunctionFactoryTest method testSomeNull.

@Test
public void testSomeNull() throws SqlException {
    compiler.compile("create table tab (f date)", sqlExecutionContext);
    try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "tab", "testing")) {
        for (int i = 100; i > 10; i--) {
            TableWriter.Row r = w.newRow();
            if (i % 4 == 0) {
                r.putLong(0, i);
            }
            r.append();
        }
        w.commit();
    }
    try (RecordCursorFactory factory = compiler.compile("select first(f) from tab", sqlExecutionContext).getRecordCursorFactory()) {
        try (RecordCursor cursor = factory.getCursor(sqlExecutionContext)) {
            Record record = cursor.getRecord();
            Assert.assertEquals(1, cursor.size());
            Assert.assertTrue(cursor.hasNext());
            Assert.assertEquals(100, record.getLong(0));
        }
    }
}