Examples with TextImportProcessor io.questdb.cutlass.http.processors.TextImportProcessor used on opensource projects

Example 1 with TextImportProcessor

use of io.questdb.cutlass.http.processors.TextImportProcessor in project questdb by bluestreak01.

the class IODispatcherTest method testImportMultipleOnSameConnectionSlow.

@Test
public void testImportMultipleOnSameConnectionSlow() throws Exception {
    assertMemoryLeak(() -> {
        final String baseDir = temp.getRoot().getAbsolutePath();
        final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1, -1, -1 };
            }

            @Override
            public int getWorkerCount() {
                return 3;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
            HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new TextImportProcessor(engine);
                }

                @Override
                public String getUrl() {
                    return "/upload";
                }
            });
            workerPool.start(LOG);
            // send multipart request to server
            final String request = "POST /upload HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "User-Agent: curl/7.64.0\r\n" + "Accept: */*\r\n" + "Content-Length: 437760673\r\n" + "Content-Type: multipart/form-data; boundary=------------------------27d997ca93d2689d\r\n" + "Expect: 100-continue\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"schema\"; filename=\"schema.json\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "[\r\n" + "  {\r\n" + "    \"name\": \"date\",\r\n" + "    \"type\": \"DATE\",\r\n" + "    \"pattern\": \"d MMMM y.\",\r\n" + "    \"locale\": \"ru-RU\"\r\n" + "  }\r\n" + "]\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"data\"; filename=\"fhv_tripdata_2017-02.csv\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "Dispatching_base_num,Pickup_DateTime,DropOff_datetime,PUlocationID,DOlocationID\r\n" + "B00008,2017-02-01 00:30:00,,,\r\n" + "B00008,2017-02-01 00:40:00,,,\r\n" + "B00009,2017-02-01 00:30:00,,,\r\n" + "B00013,2017-02-01 00:11:00,,,\r\n" + "B00013,2017-02-01 00:41:00,,,\r\n" + "B00013,2017-02-01 00:00:00,,,\r\n" + "B00013,2017-02-01 00:53:00,,,\r\n" + "B00013,2017-02-01 00:44:00,,,\r\n" + "B00013,2017-02-01 00:05:00,,,\r\n" + "B00013,2017-02-01 00:54:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "B00014,2017-02-01 00:46:00,,,\r\n" + "B00014,2017-02-01 00:54:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "B00014,2017-02-01 00:26:00,,,\r\n" + "B00014,2017-02-01 00:55:00,,,\r\n" + "B00014,2017-02-01 00:47:00,,,\r\n" + "B00014,2017-02-01 00:05:00,,,\r\n" + "B00014,2017-02-01 00:58:00,,,\r\n" + "B00014,2017-02-01 00:33:00,,,\r\n" + "B00014,2017-02-01 00:45:00,,,\r\n" + "\r\n" + "--------------------------27d997ca93d2689d--";
            NetworkFacade nf = new NetworkFacadeImpl() {

                int totalSent = 0;

                @Override
                public int send(long fd, long buffer, int bufferLen) {
                    if (bufferLen > 0) {
                        int result = super.send(fd, buffer, 1);
                        totalSent += result;
                        if (totalSent > 20) {
                            LockSupport.parkNanos(10000);
                            totalSent = 0;
                        }
                        return result;
                    }
                    return 0;
                }
            };
            try {
                sendAndReceive(nf, request, ValidImportResponse, 3, 0, false);
            } finally {
                workerPool.halt();
            }
        }
    });
}

Example 2 with TextImportProcessor

use of io.questdb.cutlass.http.processors.TextImportProcessor in project questdb by bluestreak01.

the class RetryIODispatcherTest method testFailsWhenInvalidDataImportedLoop.

@Test
public void testFailsWhenInvalidDataImportedLoop() throws Exception {
    for (int i = 0; i < 5; i++) {
        System.out.println("*************************************************************************************");
        System.out.println("**************************         Run " + i + "            ********************************");
        System.out.println("*************************************************************************************");
        testImportWaitsWhenWriterLocked(new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(2).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withCustomTextImportProcessor(((configuration, engine, workerCount) -> new TextImportProcessor(engine) {

            @Override
            public void onRequestRetry(HttpConnectionContext context) throws ServerDisconnectException {
                throw ServerDisconnectException.INSTANCE;
            }
        })), 0, ValidImportRequest, ValidImportResponse, false, true);
        temp.delete();
        temp.create();
    }
}