Examples with WorkerPool io.questdb.mp.WorkerPool used on opensource projects

Example 11 with WorkerPool

use of io.questdb.mp.WorkerPool in project questdb by bluestreak01.

the class LineTcpO3Test method test.

private void test(String ilpResourceName) throws Exception {
    assertMemoryLeak(() -> {
        long clientFd = Net.socketTcp(true);
        Assert.assertTrue(clientFd >= 0);
        long ilpSockAddr = Net.sockaddr(Net.parseIPv4("127.0.0.1"), lineConfiguration.getNetDispatcherConfiguration().getBindPort());
        WorkerPool sharedWorkerPool = new WorkerPool(sharedWorkerPoolConfiguration);
        try (LineTcpReceiver ignored = LineTcpReceiver.create(lineConfiguration, sharedWorkerPool, LOG, engine);
            SqlCompiler compiler = new SqlCompiler(engine);
            SqlExecutionContext sqlExecutionContext = new SqlExecutionContextImpl(engine, 1)) {
            SOCountDownLatch haltLatch = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN && Chars.equals(name, "cpu")) {
                    haltLatch.countDown();
                }
            });
            sharedWorkerPool.assignCleaner(Path.CLEANER);
            sharedWorkerPool.start(LOG);
            TestUtils.assertConnect(clientFd, ilpSockAddr);
            readGzResource(ilpResourceName);
            Net.send(clientFd, resourceAddress, resourceSize);
            Unsafe.free(resourceAddress, resourceSize, MemoryTag.NATIVE_DEFAULT);
            haltLatch.await();
            TestUtils.printSql(compiler, sqlExecutionContext, "select * from " + "cpu", sink);
            readGzResource("selectAll1");
            DirectUnboundedByteSink expectedSink = new DirectUnboundedByteSink(resourceAddress);
            expectedSink.clear(resourceSize);
            TestUtils.assertEquals(expectedSink.toString(), sink);
            Unsafe.free(resourceAddress, resourceSize, MemoryTag.NATIVE_DEFAULT);
        } finally {
            engine.setPoolListener(null);
            Net.close(clientFd);
            Net.freeSockAddr(ilpSockAddr);
            sharedWorkerPool.halt();
        }
    });
}

Example 12 with WorkerPool

use of io.questdb.mp.WorkerPool in project questdb by bluestreak01.

the class LineTcpReceiver method create.

@Nullable
public static LineTcpReceiver create(LineTcpReceiverConfiguration lineConfiguration, WorkerPool sharedWorkerPool, Log log, CairoEngine cairoEngine) {
    if (!lineConfiguration.isEnabled()) {
        return null;
    }
    ObjList<WorkerPool> dedicatedPools = new ObjList<>(2);
    WorkerPool ioWorkerPool = WorkerPoolAwareConfiguration.configureWorkerPool(lineConfiguration.getIOWorkerPoolConfiguration(), sharedWorkerPool);
    WorkerPool writerWorkerPool = WorkerPoolAwareConfiguration.configureWorkerPool(lineConfiguration.getWriterWorkerPoolConfiguration(), sharedWorkerPool);
    if (ioWorkerPool != sharedWorkerPool) {
        ioWorkerPool.assignCleaner(Path.CLEANER);
        dedicatedPools.add(ioWorkerPool);
    }
    if (writerWorkerPool != sharedWorkerPool) {
        writerWorkerPool.assignCleaner(Path.CLEANER);
        dedicatedPools.add(writerWorkerPool);
    }
    LineTcpReceiver lineTcpReceiver = new LineTcpReceiver(lineConfiguration, cairoEngine, ioWorkerPool, writerWorkerPool, dedicatedPools);
    if (ioWorkerPool != sharedWorkerPool) {
        ioWorkerPool.start(log);
    }
    if (writerWorkerPool != sharedWorkerPool) {
        writerWorkerPool.start(log);
    }
    return lineTcpReceiver;
}

Example 13 with WorkerPool

use of io.questdb.mp.WorkerPool in project questdb by bluestreak01.

the class IODispatcherTest method testJsonQueryWithInterruption.

@Test
public void testJsonQueryWithInterruption() throws Exception {
    assertMemoryLeak(() -> {
        final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
        final String baseDir = temp.getRoot().getAbsolutePath();
        final int tableRowCount = 300_000;
        SOCountDownLatch peerDisconnectLatch = new SOCountDownLatch(1);
        DefaultHttpServerConfiguration httpConfiguration = new HttpServerConfigurationBuilder().withNetwork(nf).withBaseDir(baseDir).withSendBufferSize(256).withDumpingTraffic(false).withAllowDeflateBeforeSend(false).withServerKeepAlive(true).withHttpProtocolVersion("HTTP/1.1 ").withOnPeerDisconnect(peerDisconnectLatch::countDown).build();
        QueryCache.configure(httpConfiguration);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
            HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
                }

                @Override
                public String getUrl() {
                    return "/query";
                }
            });
            final int minClientReceivedBytesBeforeDisconnect = 180;
            final AtomicLong refClientFd = new AtomicLong(-1);
            HttpClientStateListener clientStateListener = new HttpClientStateListener() {

                private int nBytesReceived = 0;

                @Override
                public void onClosed() {
                }

                @Override
                public void onReceived(int nBytes) {
                    LOG.info().$("Client received ").$(nBytes).$(" bytes").$();
                    nBytesReceived += nBytes;
                    if (nBytesReceived >= minClientReceivedBytesBeforeDisconnect) {
                        long fd = refClientFd.get();
                        if (fd != -1) {
                            refClientFd.set(-1);
                            nf.close(fd);
                        }
                    }
                }
            };
            workerPool.start(LOG);
            try {
                // create table with all column types
                CairoTestUtils.createTestTable(engine.getConfiguration(), tableRowCount, new Rnd(), new TestRecord.ArrayBinarySequence());
                // send multipart request to server
                final String request = "GET /query?query=select+distinct+a+from+x+where+test_latched_counter() HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
                long fd = nf.socketTcp(true);
                try {
                    long sockAddr = nf.sockaddr("127.0.0.1", 9001);
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        Assert.assertEquals(0, nf.setTcpNoDelay(fd, true));
                        refClientFd.set(fd);
                        nf.configureNonBlocking(fd);
                        long bufLen = request.length();
                        long ptr = Unsafe.malloc(bufLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            new SendAndReceiveRequestBuilder().withNetworkFacade(nf).withPauseBetweenSendAndReceive(0).withPrintOnly(false).withExpectDisconnect(true).executeUntilDisconnect(request, fd, 200, ptr, clientStateListener);
                        } finally {
                            Unsafe.free(ptr, bufLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        nf.freeSockAddr(sockAddr);
                    }
                } finally {
                    LOG.info().$("Closing client connection").$();
                }
                peerDisconnectLatch.await();
                // depending on how quick the CI hardware is we may end up processing different
                // number of rows before query is interrupted
                Assert.assertTrue(tableRowCount > TestLatchedCounterFunctionFactory.getCount());
            } finally {
                workerPool.halt();
            }
        }
    });
}

Example 14 with WorkerPool

use of io.questdb.mp.WorkerPool in project questdb by bluestreak01.

the class IODispatcherTest method testSCPHttp10.

@Test
public void testSCPHttp10() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(NetworkFacadeImpl.INSTANCE, baseDir, 16 * 1024, false, false, false, "HTTP/1.0 ");
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 2;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            workerPool.start(LOG);
            // create 20Mb file in /tmp directory
            try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
                try {
                    Rnd rnd = new Rnd();
                    final int diskBufferLen = 1024 * 1024;
                    writeRandomFile(path, rnd, 122222212222L);
                    long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                    try {
                        int netBufferLen = 4 * 1024;
                        long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            // send request to server to download file we just created
                            final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader = "HTTP/1.0 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + "Connection: close\r\n" + // this is last modified timestamp on the file, we set this value when we created file
                            "ETag: \"122222212222\"\r\n" + "\r\n";
                            for (int j = 0; j < 1; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // send few requests to receive 304
                            final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
                            "If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader2 = "HTTP/1.0 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n";
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // couple more full downloads after 304
                            for (int j = 0; j < 2; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // get a 404 now
                            final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader3 = "HTTP/1.0 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
                            for (int i = 0; i < 4; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request3, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader3, expectedResponseHeader3.length());
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                        } finally {
                            Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        Net.freeSockAddr(sockAddr);
                        workerPool.halt();
                    }
                } finally {
                    Files.remove(path);
                }
            }
        }
    });
}

Example 15 with WorkerPool

use of io.questdb.mp.WorkerPool in project questdb by bluestreak01.

the class IODispatcherTest method testSCPConnectDownloadDisconnect.

@Test
public void testSCPConnectDownloadDisconnect() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 2;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            workerPool.start(LOG);
            // create 20Mb file in /tmp directory
            try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
                try {
                    Rnd rnd = new Rnd();
                    final int diskBufferLen = 1024 * 1024;
                    writeRandomFile(path, rnd, 122222212222L);
                    // httpServer.getStartedLatch().await();
                    long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                    try {
                        int netBufferLen = 4 * 1024;
                        long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            // send request to server to download file we just created
                            final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + // this is last modified timestamp on the file, we set this value when we created file
                            "ETag: \"122222212222\"\r\n" + "\r\n";
                            for (int j = 0; j < 10; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // send few requests to receive 304
                            final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
                            "If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader2 = "HTTP/1.1 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "\r\n";
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // couple more full downloads after 304
                            for (int j = 0; j < 2; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // get a 404 now
                            final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader3 = "HTTP/1.1 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
                            sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false);
                            // and few more 304s
                            sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false);
                        } finally {
                            Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        Net.freeSockAddr(sockAddr);
                    }
                } finally {
                    workerPool.halt();
                    Files.remove(path);
                }
            }
        }
    });
}