use of io.questdb.cutlass.http.processors.JsonQueryProcessor in project questdb by bluestreak01.
the class IODispatcherTest method testJsonQuerySyntaxError.
@Test
public void testJsonQuerySyntaxError() throws Exception {
assertMemoryLeak(() -> {
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1 };
}
@Override
public int getWorkerCount() {
return 1;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
}
@Override
public String getUrl() {
return "/query";
}
});
workerPool.start(LOG);
try {
// create table with all column types
CairoTestUtils.createTestTable(engine.getConfiguration(), 20, new Rnd(), new TestRecord.ArrayBinarySequence());
// send multipart request to server
final String request = "GET /query?query=x%20where2%20i%20%3D%20(%27EHNRX%27) HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
String expectedResponse = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: application/json; charset=utf-8\r\n" + "Keep-Alive: timeout=5, max=10000\r\n" + "\r\n" + "4d\r\n" + "{\"query\":\"x where2 i = ('EHNRX')\",\"error\":\"unexpected token: i\",\"position\":9}\r\n" + "00\r\n" + "\r\n";
sendAndReceive(NetworkFacadeImpl.INSTANCE, request, expectedResponse, 10, 0, false);
} finally {
workerPool.halt();
}
}
});
}
use of io.questdb.cutlass.http.processors.JsonQueryProcessor in project questdb by bluestreak01.
the class IODispatcherTest method testJsonQueryWithCompressedResults1.
@Test
public void testJsonQueryWithCompressedResults1() throws Exception {
Zip.init();
assertMemoryLeak(() -> {
final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(nf, baseDir, 256, false, true);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1, -1 };
}
@Override
public int getWorkerCount() {
return 2;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
}
@Override
public String getUrl() {
return "/query";
}
});
workerPool.start(LOG);
try {
// create table with all column types
CairoTestUtils.createTestTable(engine.getConfiguration(), 30, new Rnd(), new TestRecord.ArrayBinarySequence());
// send multipart request to server
final String request = "GET /query?query=x HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
ByteArrayResponse expectedResponse;
try (InputStream is = getClass().getResourceAsStream(getClass().getSimpleName() + ".testJsonQueryWithCompressedResults1.bin")) {
Assert.assertNotNull(is);
byte[] bytes = new byte[20 * 1024];
int len = is.read(bytes);
expectedResponse = new ByteArrayResponse(bytes, len);
}
sendAndReceive(nf, request, expectedResponse, 10, 100L, false);
} finally {
workerPool.halt();
}
}
});
}
use of io.questdb.cutlass.http.processors.JsonQueryProcessor in project questdb by bluestreak01.
the class IODispatcherTest method testJsonQueryDataError.
@Test
public void testJsonQueryDataError() throws Exception {
assertMemoryLeak(() -> {
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1 };
}
@Override
public int getWorkerCount() {
return 1;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
}
@Override
public String getUrl() {
return "/query";
}
});
workerPool.start(LOG);
try {
// send multipart request to server
final String request = "GET /query?limit=0%2C1000&count=true&src=con&query=select%20npe()%20from%20long_sequence(1)&timings=true HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\r\n" + "Accept: */*\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + "Sec-Fetch-Dest: empty\r\n" + "Referer: http://localhost:9000/index.html\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "Cookie: _ga=GA1.1.2124932001.1573824669; _hjid=f2db90b2-18cf-4956-8870-fcdcde56f3ca; _hjIncludedInSample=1; _gid=GA1.1.697400188.1591597903\r\n" + "\r\n";
long fd = NetworkFacadeImpl.INSTANCE.socketTcp(true);
try {
long sockAddr = NetworkFacadeImpl.INSTANCE.sockaddr("127.0.0.1", 9001);
try {
TestUtils.assertConnect(fd, sockAddr);
Assert.assertEquals(0, NetworkFacadeImpl.INSTANCE.setTcpNoDelay(fd, true));
final int len = request.length() * 2;
long ptr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
try {
int sent = 0;
int reqLen = request.length();
Chars.asciiStrCpy(request, reqLen, ptr);
while (sent < reqLen) {
int n = NetworkFacadeImpl.INSTANCE.send(fd, ptr + sent, reqLen - sent);
Assert.assertTrue(n > -1);
sent += n;
}
Os.sleep(1);
NetworkFacadeImpl.INSTANCE.configureNonBlocking(fd);
long t = System.currentTimeMillis();
boolean disconnected = true;
while (NetworkFacadeImpl.INSTANCE.recv(fd, ptr, 1) > -1) {
if (t + 20000 < System.currentTimeMillis()) {
disconnected = false;
break;
}
}
Assert.assertTrue("disconnect expected", disconnected);
} finally {
Unsafe.free(ptr, len, MemoryTag.NATIVE_DEFAULT);
}
} finally {
NetworkFacadeImpl.INSTANCE.freeSockAddr(sockAddr);
}
} finally {
NetworkFacadeImpl.INSTANCE.close(fd);
}
} finally {
workerPool.halt();
}
}
});
}
use of io.questdb.cutlass.http.processors.JsonQueryProcessor in project questdb by bluestreak01.
the class IODispatcherTest method testJsonQueryWithInterruption.
@Test
public void testJsonQueryWithInterruption() throws Exception {
assertMemoryLeak(() -> {
final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
final String baseDir = temp.getRoot().getAbsolutePath();
final int tableRowCount = 300_000;
SOCountDownLatch peerDisconnectLatch = new SOCountDownLatch(1);
DefaultHttpServerConfiguration httpConfiguration = new HttpServerConfigurationBuilder().withNetwork(nf).withBaseDir(baseDir).withSendBufferSize(256).withDumpingTraffic(false).withAllowDeflateBeforeSend(false).withServerKeepAlive(true).withHttpProtocolVersion("HTTP/1.1 ").withOnPeerDisconnect(peerDisconnectLatch::countDown).build();
QueryCache.configure(httpConfiguration);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1 };
}
@Override
public int getWorkerCount() {
return 1;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
}
@Override
public String getUrl() {
return "/query";
}
});
final int minClientReceivedBytesBeforeDisconnect = 180;
final AtomicLong refClientFd = new AtomicLong(-1);
HttpClientStateListener clientStateListener = new HttpClientStateListener() {
private int nBytesReceived = 0;
@Override
public void onClosed() {
}
@Override
public void onReceived(int nBytes) {
LOG.info().$("Client received ").$(nBytes).$(" bytes").$();
nBytesReceived += nBytes;
if (nBytesReceived >= minClientReceivedBytesBeforeDisconnect) {
long fd = refClientFd.get();
if (fd != -1) {
refClientFd.set(-1);
nf.close(fd);
}
}
}
};
workerPool.start(LOG);
try {
// create table with all column types
CairoTestUtils.createTestTable(engine.getConfiguration(), tableRowCount, new Rnd(), new TestRecord.ArrayBinarySequence());
// send multipart request to server
final String request = "GET /query?query=select+distinct+a+from+x+where+test_latched_counter() HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
long fd = nf.socketTcp(true);
try {
long sockAddr = nf.sockaddr("127.0.0.1", 9001);
try {
TestUtils.assertConnect(fd, sockAddr);
Assert.assertEquals(0, nf.setTcpNoDelay(fd, true));
refClientFd.set(fd);
nf.configureNonBlocking(fd);
long bufLen = request.length();
long ptr = Unsafe.malloc(bufLen, MemoryTag.NATIVE_DEFAULT);
try {
new SendAndReceiveRequestBuilder().withNetworkFacade(nf).withPauseBetweenSendAndReceive(0).withPrintOnly(false).withExpectDisconnect(true).executeUntilDisconnect(request, fd, 200, ptr, clientStateListener);
} finally {
Unsafe.free(ptr, bufLen, MemoryTag.NATIVE_DEFAULT);
}
} finally {
nf.freeSockAddr(sockAddr);
}
} finally {
LOG.info().$("Closing client connection").$();
}
peerDisconnectLatch.await();
// depending on how quick the CI hardware is we may end up processing different
// number of rows before query is interrupted
Assert.assertTrue(tableRowCount > TestLatchedCounterFunctionFactory.getCount());
} finally {
workerPool.halt();
}
}
});
}
use of io.questdb.cutlass.http.processors.JsonQueryProcessor in project questdb by bluestreak01.
the class IODispatcherTest method testJsonQueryWithCompressedResults2.
@Test
public void testJsonQueryWithCompressedResults2() throws Exception {
Zip.init();
assertMemoryLeak(() -> {
final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(nf, baseDir, 4096, false, true);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1, -1 };
}
@Override
public int getWorkerCount() {
return 2;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
}
@Override
public String getUrl() {
return "/query";
}
});
workerPool.start(LOG);
try {
// create table with all column types
CairoTestUtils.createTestTable(engine.getConfiguration(), 1000, new Rnd(), new TestRecord.ArrayBinarySequence());
// send multipart request to server
final String request = "GET /query?query=x HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
ByteArrayResponse expectedResponse;
try (InputStream is = getClass().getResourceAsStream(getClass().getSimpleName() + ".testJsonQueryWithCompressedResults2.bin")) {
Assert.assertNotNull(is);
byte[] bytes = new byte[100 * 1024];
int len = is.read(bytes);
expectedResponse = new ByteArrayResponse(bytes, len);
}
sendAndReceive(nf, request, expectedResponse, 10, 100L, false);
} finally {
workerPool.halt();
}
}
});
}
Aggregations