Examples with WorkerPoolConfiguration io.questdb.mp.WorkerPoolConfiguration used on opensource projects

Example 6 with WorkerPoolConfiguration

use of io.questdb.mp.WorkerPoolConfiguration in project questdb by bluestreak01.

the class IODispatcherTest method testSCPFullDownload.

@Test
public void testSCPFullDownload() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 2;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            workerPool.start(LOG);
            // create 20Mb file in /tmp directory
            try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
                try {
                    Rnd rnd = new Rnd();
                    final int diskBufferLen = 1024 * 1024;
                    writeRandomFile(path, rnd, 122299092L);
                    long fd = Net.socketTcp(true);
                    try {
                        long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                        try {
                            TestUtils.assertConnect(fd, sockAddr);
                            int netBufferLen = 4 * 1024;
                            long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
                            try {
                                // send request to server to download file we just created
                                final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                                String expectedResponseHeader = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + // this is last modified timestamp on the file, we set this value when we created file
                                "ETag: \"122299092\"\r\n" + "\r\n";
                                for (int j = 0; j < 10; j++) {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971667);
                                }
                                // send few requests to receive 304
                                final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
                                "If-None-Match: \"122299092\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                                String expectedResponseHeader2 = "HTTP/1.1 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "\r\n";
                                for (int i = 0; i < 3; i++) {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                }
                                // couple more full downloads after 304
                                for (int j = 0; j < 2; j++) {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971667);
                                }
                                // get a 404 now
                                final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                                String expectedResponseHeader3 = "HTTP/1.1 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
                                sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false);
                                // and few more 304s
                                sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false);
                            } finally {
                                Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
                            }
                        } finally {
                            Net.freeSockAddr(sockAddr);
                        }
                    } finally {
                        Net.close(fd);
                        LOG.info().$("closed [fd=").$(fd).$(']').$();
                    }
                } finally {
                    workerPool.halt();
                    Files.remove(path);
                }
            }
        }
    });
}

Example 7 with WorkerPoolConfiguration

use of io.questdb.mp.WorkerPoolConfiguration in project questdb by bluestreak01.

the class IODispatcherTest method testJsonQueryDataError.

@Test
public void testJsonQueryDataError() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
            HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
                }

                @Override
                public String getUrl() {
                    return "/query";
                }
            });
            workerPool.start(LOG);
            try {
                // send multipart request to server
                final String request = "GET /query?limit=0%2C1000&count=true&src=con&query=select%20npe()%20from%20long_sequence(1)&timings=true HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\r\n" + "Accept: */*\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + "Sec-Fetch-Dest: empty\r\n" + "Referer: http://localhost:9000/index.html\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "Cookie: _ga=GA1.1.2124932001.1573824669; _hjid=f2db90b2-18cf-4956-8870-fcdcde56f3ca; _hjIncludedInSample=1; _gid=GA1.1.697400188.1591597903\r\n" + "\r\n";
                long fd = NetworkFacadeImpl.INSTANCE.socketTcp(true);
                try {
                    long sockAddr = NetworkFacadeImpl.INSTANCE.sockaddr("127.0.0.1", 9001);
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        Assert.assertEquals(0, NetworkFacadeImpl.INSTANCE.setTcpNoDelay(fd, true));
                        final int len = request.length() * 2;
                        long ptr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
                        try {
                            int sent = 0;
                            int reqLen = request.length();
                            Chars.asciiStrCpy(request, reqLen, ptr);
                            while (sent < reqLen) {
                                int n = NetworkFacadeImpl.INSTANCE.send(fd, ptr + sent, reqLen - sent);
                                Assert.assertTrue(n > -1);
                                sent += n;
                            }
                            Os.sleep(1);
                            NetworkFacadeImpl.INSTANCE.configureNonBlocking(fd);
                            long t = System.currentTimeMillis();
                            boolean disconnected = true;
                            while (NetworkFacadeImpl.INSTANCE.recv(fd, ptr, 1) > -1) {
                                if (t + 20000 < System.currentTimeMillis()) {
                                    disconnected = false;
                                    break;
                                }
                            }
                            Assert.assertTrue("disconnect expected", disconnected);
                        } finally {
                            Unsafe.free(ptr, len, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        NetworkFacadeImpl.INSTANCE.freeSockAddr(sockAddr);
                    }
                } finally {
                    NetworkFacadeImpl.INSTANCE.close(fd);
                }
            } finally {
                workerPool.halt();
            }
        }
    });
}

Example 8 with WorkerPoolConfiguration

use of io.questdb.mp.WorkerPoolConfiguration in project questdb by bluestreak01.

the class IODispatcherTest method testJsonQueryWithInterruption.

@Test
public void testJsonQueryWithInterruption() throws Exception {
    assertMemoryLeak(() -> {
        final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
        final String baseDir = temp.getRoot().getAbsolutePath();
        final int tableRowCount = 300_000;
        SOCountDownLatch peerDisconnectLatch = new SOCountDownLatch(1);
        DefaultHttpServerConfiguration httpConfiguration = new HttpServerConfigurationBuilder().withNetwork(nf).withBaseDir(baseDir).withSendBufferSize(256).withDumpingTraffic(false).withAllowDeflateBeforeSend(false).withServerKeepAlive(true).withHttpProtocolVersion("HTTP/1.1 ").withOnPeerDisconnect(peerDisconnectLatch::countDown).build();
        QueryCache.configure(httpConfiguration);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
            HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
                }

                @Override
                public String getUrl() {
                    return "/query";
                }
            });
            final int minClientReceivedBytesBeforeDisconnect = 180;
            final AtomicLong refClientFd = new AtomicLong(-1);
            HttpClientStateListener clientStateListener = new HttpClientStateListener() {

                private int nBytesReceived = 0;

                @Override
                public void onClosed() {
                }

                @Override
                public void onReceived(int nBytes) {
                    LOG.info().$("Client received ").$(nBytes).$(" bytes").$();
                    nBytesReceived += nBytes;
                    if (nBytesReceived >= minClientReceivedBytesBeforeDisconnect) {
                        long fd = refClientFd.get();
                        if (fd != -1) {
                            refClientFd.set(-1);
                            nf.close(fd);
                        }
                    }
                }
            };
            workerPool.start(LOG);
            try {
                // create table with all column types
                CairoTestUtils.createTestTable(engine.getConfiguration(), tableRowCount, new Rnd(), new TestRecord.ArrayBinarySequence());
                // send multipart request to server
                final String request = "GET /query?query=select+distinct+a+from+x+where+test_latched_counter() HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
                long fd = nf.socketTcp(true);
                try {
                    long sockAddr = nf.sockaddr("127.0.0.1", 9001);
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        Assert.assertEquals(0, nf.setTcpNoDelay(fd, true));
                        refClientFd.set(fd);
                        nf.configureNonBlocking(fd);
                        long bufLen = request.length();
                        long ptr = Unsafe.malloc(bufLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            new SendAndReceiveRequestBuilder().withNetworkFacade(nf).withPauseBetweenSendAndReceive(0).withPrintOnly(false).withExpectDisconnect(true).executeUntilDisconnect(request, fd, 200, ptr, clientStateListener);
                        } finally {
                            Unsafe.free(ptr, bufLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        nf.freeSockAddr(sockAddr);
                    }
                } finally {
                    LOG.info().$("Closing client connection").$();
                }
                peerDisconnectLatch.await();
                // depending on how quick the CI hardware is we may end up processing different
                // number of rows before query is interrupted
                Assert.assertTrue(tableRowCount > TestLatchedCounterFunctionFactory.getCount());
            } finally {
                workerPool.halt();
            }
        }
    });
}

Example 9 with WorkerPoolConfiguration

use of io.questdb.mp.WorkerPoolConfiguration in project questdb by bluestreak01.

the class IODispatcherTest method testSCPHttp10.

@Test
public void testSCPHttp10() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(NetworkFacadeImpl.INSTANCE, baseDir, 16 * 1024, false, false, false, "HTTP/1.0 ");
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 2;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            workerPool.start(LOG);
            // create 20Mb file in /tmp directory
            try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
                try {
                    Rnd rnd = new Rnd();
                    final int diskBufferLen = 1024 * 1024;
                    writeRandomFile(path, rnd, 122222212222L);
                    long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                    try {
                        int netBufferLen = 4 * 1024;
                        long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            // send request to server to download file we just created
                            final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader = "HTTP/1.0 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + "Connection: close\r\n" + // this is last modified timestamp on the file, we set this value when we created file
                            "ETag: \"122222212222\"\r\n" + "\r\n";
                            for (int j = 0; j < 1; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // send few requests to receive 304
                            final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
                            "If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader2 = "HTTP/1.0 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n";
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // couple more full downloads after 304
                            for (int j = 0; j < 2; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // get a 404 now
                            final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader3 = "HTTP/1.0 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
                            for (int i = 0; i < 4; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request3, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader3, expectedResponseHeader3.length());
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                        } finally {
                            Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        Net.freeSockAddr(sockAddr);
                        workerPool.halt();
                    }
                } finally {
                    Files.remove(path);
                }
            }
        }
    });
}

Example 10 with WorkerPoolConfiguration

use of io.questdb.mp.WorkerPoolConfiguration in project questdb by bluestreak01.

the class IODispatcherTest method testSCPConnectDownloadDisconnect.

@Test
public void testSCPConnectDownloadDisconnect() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 2;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            workerPool.start(LOG);
            // create 20Mb file in /tmp directory
            try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
                try {
                    Rnd rnd = new Rnd();
                    final int diskBufferLen = 1024 * 1024;
                    writeRandomFile(path, rnd, 122222212222L);
                    // httpServer.getStartedLatch().await();
                    long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                    try {
                        int netBufferLen = 4 * 1024;
                        long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            // send request to server to download file we just created
                            final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + // this is last modified timestamp on the file, we set this value when we created file
                            "ETag: \"122222212222\"\r\n" + "\r\n";
                            for (int j = 0; j < 10; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // send few requests to receive 304
                            final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
                            "If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader2 = "HTTP/1.1 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "\r\n";
                            for (int i = 0; i < 3; i++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request2, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // couple more full downloads after 304
                            for (int j = 0; j < 2; j++) {
                                long fd = Net.socketTcp(true);
                                TestUtils.assertConnect(fd, sockAddr);
                                try {
                                    sendRequest(request, fd, buffer);
                                    assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
                                } finally {
                                    Net.close(fd);
                                }
                            }
                            // get a 404 now
                            final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                            String expectedResponseHeader3 = "HTTP/1.1 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
                            sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false);
                            // and few more 304s
                            sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false);
                        } finally {
                            Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        Net.freeSockAddr(sockAddr);
                    }
                } finally {
                    workerPool.halt();
                    Files.remove(path);
                }
            }
        }
    });
}