Examples with SOCountDownLatch io.questdb.mp.SOCountDownLatch used on opensource projects

Example 6 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class IODispatcherTest method testMaxConnections.

@Test
public void testMaxConnections() throws Exception {
    LOG.info().$("started maxConnections").$();
    assertMemoryLeak(() -> {
        HttpServerConfiguration httpServerConfiguration = new DefaultHttpServerConfiguration();
        final int listenBackLog = 10;
        final int activeConnectionLimit = 5;
        final int nExcessConnections = 20;
        AtomicInteger openCount = new AtomicInteger(0);
        AtomicInteger closeCount = new AtomicInteger(0);
        final IODispatcherConfiguration configuration = new DefaultIODispatcherConfiguration() {

            @Override
            public int getActiveConnectionLimit() {
                return activeConnectionLimit;
            }

            @Override
            public int getListenBacklog() {
                return listenBackLog;
            }

            @Override
            public long getQueuedConnectionTimeout() {
                return 300_000;
            }
        };
        try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(configuration, new IOContextFactory<HttpConnectionContext>() {

            @SuppressWarnings("resource")
            @Override
            public HttpConnectionContext newInstance(long fd, IODispatcher<HttpConnectionContext> dispatcher1) {
                openCount.incrementAndGet();
                return new HttpConnectionContext(httpServerConfiguration.getHttpContextConfiguration()) {

                    @Override
                    public void close() {
                        closeCount.incrementAndGet();
                        super.close();
                    }
                }.of(fd, dispatcher1);
            }
        })) {
            HttpRequestProcessorSelector selector = new HttpRequestProcessorSelector() {

                @Override
                public HttpRequestProcessor select(CharSequence url) {
                    return null;
                }

                @Override
                public HttpRequestProcessor getDefaultProcessor() {
                    return new HealthCheckProcessor();
                }

                @Override
                public void close() {
                }
            };
            AtomicBoolean serverRunning = new AtomicBoolean(true);
            SOCountDownLatch serverHaltLatch = new SOCountDownLatch(1);
            new Thread(() -> {
                try {
                    do {
                        dispatcher.run(0);
                        dispatcher.processIOQueue((operation, context) -> context.handleClientOperation(operation, selector, EmptyRescheduleContext));
                    } while (serverRunning.get());
                } finally {
                    serverHaltLatch.countDown();
                }
            }).start();
            LongList openFds = new LongList();
            LongHashSet closedFds = new LongHashSet();
            final long sockAddr = Net.sockaddr("127.0.0.1", 9001);
            final long buf = Unsafe.malloc(4096, MemoryTag.NATIVE_DEFAULT);
            final int N = activeConnectionLimit + listenBackLog;
            try {
                for (int i = 0; i < N; i++) {
                    long fd = Net.socketTcp(true);
                    TestUtils.assertConnect(fd, sockAddr);
                    openFds.add(fd);
                }
                // let dispatcher catchup
                long startNanos = System.nanoTime();
                while (dispatcher.isListening()) {
                    long endNanos = System.nanoTime();
                    if (TimeUnit.NANOSECONDS.toSeconds(endNanos - startNanos) > 30) {
                        Assert.fail("Timed out waiting for despatcher to stop listening");
                    }
                    LockSupport.parkNanos(1);
                }
                final String request = "GET /status?x=1&a=%26b&c&d=x HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
                long mem = TestUtils.toMemory(request);
                // Active connection limit is reached and backlog is full, check we get connection closed
                for (int i = 0; i < nExcessConnections; i++) {
                    long fd = Net.socketTcp(true);
                    if (fd > 0) {
                        int nSent = Net.send(fd, mem, request.length());
                        Assert.assertTrue(nSent < 0);
                        Net.close(fd);
                    }
                }
                try {
                    for (int i = 0; i < N; i++) {
                        if (i == activeConnectionLimit) {
                            for (int j = 0; j < activeConnectionLimit; j++) {
                                long fd = openFds.getQuick(j);
                                Net.close(fd);
                                closedFds.add(fd);
                            }
                        }
                        long fd = openFds.getQuick(i);
                        Assert.assertEquals(request.length(), Net.send(fd, mem, request.length()));
                        // ensure we have response from server
                        Assert.assertTrue(0 < Net.recv(fd, buf, 64));
                        if (i < activeConnectionLimit) {
                            // dont close any connections until the first connections have been processed and check that the dispatcher
                            // is not listening
                            Assert.assertFalse(dispatcher.isListening());
                        } else {
                            Net.close(fd);
                            closedFds.add(fd);
                        }
                    }
                } finally {
                    Unsafe.free(mem, request.length(), MemoryTag.NATIVE_DEFAULT);
                }
            } finally {
                for (int i = 0; i < openFds.size(); i++) {
                    long fd = openFds.getQuick(i);
                    if (!closedFds.contains(fd)) {
                        Net.close(fd);
                    }
                }
                Net.freeSockAddr(sockAddr);
                Unsafe.free(buf, 4096, MemoryTag.NATIVE_DEFAULT);
                Assert.assertFalse(configuration.getActiveConnectionLimit() < dispatcher.getConnectionCount());
                serverRunning.set(false);
                serverHaltLatch.await();
            }
        }
    });
}

Example 7 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class IODispatcherTest method testCannotSetNonBlocking.

@Test
public void testCannotSetNonBlocking() throws Exception {
    assertMemoryLeak(() -> {
        final HttpContextConfiguration httpContextConfiguration = new DefaultHttpContextConfiguration();
        final NetworkFacade nf = new NetworkFacadeImpl() {

            long theFd;

            @Override
            public long accept(long serverFd) {
                long fd = super.accept(serverFd);
                theFd = fd;
                return fd;
            }

            @Override
            public int configureNonBlocking(long fd) {
                if (fd == theFd) {
                    return -1;
                }
                return super.configureNonBlocking(fd);
            }
        };
        try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration() {

            @Override
            public NetworkFacade getNetworkFacade() {
                return nf;
            }
        }, (fd, dispatcher1) -> new HttpConnectionContext(httpContextConfiguration).of(fd, dispatcher1))) {
            // spin up dispatcher thread
            AtomicBoolean dispatcherRunning = new AtomicBoolean(true);
            SOCountDownLatch dispatcherHaltLatch = new SOCountDownLatch(1);
            new Thread(() -> {
                while (dispatcherRunning.get()) {
                    dispatcher.run(0);
                }
                dispatcherHaltLatch.countDown();
            }).start();
            try {
                long socketAddr = Net.sockaddr(Net.parseIPv4("127.0.0.1"), 9001);
                long fd = Net.socketTcp(true);
                try {
                    TestUtils.assertConnect(fd, socketAddr);
                    int bufLen = 512;
                    long mem = Unsafe.malloc(bufLen, MemoryTag.NATIVE_DEFAULT);
                    try {
                        Assert.assertEquals(-2, Net.recv(fd, mem, bufLen));
                    } finally {
                        Unsafe.free(mem, bufLen, MemoryTag.NATIVE_DEFAULT);
                    }
                } finally {
                    Net.close(fd);
                    Net.freeSockAddr(socketAddr);
                }
            } finally {
                dispatcherRunning.set(false);
                dispatcherHaltLatch.await();
            }
        }
    });
}

Example 8 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class IODispatcherTest method testTwoThreadsSendTwoThreadsRead.

@Test
public // dispatcher or Http parser.
void testTwoThreadsSendTwoThreadsRead() throws Exception {
    LOG.info().$("started testSendHttpGet").$();
    final String request = "GET /status?x=1&a=%26b&c&d=x HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
    // the difference between request and expected is url encoding (and ':' padding, which can easily be fixed)
    final String expected = "GET /status?x=1&a=&b&c&d=x HTTP/1.1\r\n" + "host:localhost:9000\r\n" + "connection:keep-alive\r\n" + "cache-control:max-age=0\r\n" + "accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "user-agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "accept-encoding:gzip,deflate,sdch\r\n" + "accept-language:en-US,en;q=0.8\r\n" + "cookie:textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
    final int N = 100;
    final int serverThreadCount = 2;
    final int senderCount = 2;
    assertMemoryLeak(() -> {
        HttpServerConfiguration httpServerConfiguration = new DefaultHttpServerConfiguration();
        final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
        final AtomicInteger requestsReceived = new AtomicInteger();
        final AtomicBoolean finished = new AtomicBoolean(false);
        final SOCountDownLatch senderHalt = new SOCountDownLatch(senderCount);
        try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration(), (fd, dispatcher1) -> new HttpConnectionContext(httpServerConfiguration.getHttpContextConfiguration()).of(fd, dispatcher1))) {
            // server will publish status of each request to this queue
            final RingQueue<Status> queue = new RingQueue<>(Status::new, 1024);
            final MPSequence pubSeq = new MPSequence(queue.getCycle());
            SCSequence subSeq = new SCSequence();
            pubSeq.then(subSeq).then(pubSeq);
            final AtomicBoolean serverRunning = new AtomicBoolean(true);
            final SOCountDownLatch serverHaltLatch = new SOCountDownLatch(serverThreadCount);
            try {
                for (int j = 0; j < serverThreadCount; j++) {
                    new Thread(() -> {
                        final StringSink sink = new StringSink();
                        final long responseBuf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT);
                        Unsafe.getUnsafe().putByte(responseBuf, (byte) 'A');
                        final HttpRequestProcessor processor = new HttpRequestProcessor() {

                            @Override
                            public void onHeadersReady(HttpConnectionContext context) {
                                HttpRequestHeader headers = context.getRequestHeader();
                                sink.clear();
                                sink.put(headers.getMethodLine());
                                sink.put("\r\n");
                                ObjList<CharSequence> headerNames = headers.getHeaderNames();
                                for (int i = 0, n = headerNames.size(); i < n; i++) {
                                    sink.put(headerNames.getQuick(i)).put(':');
                                    sink.put(headers.getHeader(headerNames.getQuick(i)));
                                    sink.put("\r\n");
                                }
                                sink.put("\r\n");
                                boolean result;
                                try {
                                    TestUtils.assertEquals(expected, sink);
                                    result = true;
                                } catch (Exception e) {
                                    result = false;
                                }
                                while (true) {
                                    long cursor = pubSeq.next();
                                    if (cursor < 0) {
                                        continue;
                                    }
                                    queue.get(cursor).valid = result;
                                    pubSeq.done(cursor);
                                    break;
                                }
                                requestsReceived.incrementAndGet();
                                nf.send(context.getFd(), responseBuf, 1);
                            }
                        };
                        HttpRequestProcessorSelector selector = new HttpRequestProcessorSelector() {

                            @Override
                            public HttpRequestProcessor select(CharSequence url) {
                                return null;
                            }

                            @Override
                            public HttpRequestProcessor getDefaultProcessor() {
                                return processor;
                            }

                            @Override
                            public void close() {
                            }
                        };
                        while (serverRunning.get()) {
                            dispatcher.run(0);
                            dispatcher.processIOQueue((operation, context) -> context.handleClientOperation(operation, selector, EmptyRescheduleContext));
                        }
                        Unsafe.free(responseBuf, 32, MemoryTag.NATIVE_DEFAULT);
                        serverHaltLatch.countDown();
                    }).start();
                }
                AtomicInteger completedCount = new AtomicInteger();
                for (int j = 0; j < senderCount; j++) {
                    int k = j;
                    new Thread(() -> {
                        long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                        try {
                            for (int i = 0; i < N && !finished.get(); i++) {
                                long fd = Net.socketTcp(true);
                                try {
                                    TestUtils.assertConnect(fd, sockAddr);
                                    int len = request.length();
                                    long buffer = TestUtils.toMemory(request);
                                    try {
                                        Assert.assertEquals(len, Net.send(fd, buffer, len));
                                        Assert.assertEquals("fd=" + fd + ", i=" + i, 1, Net.recv(fd, buffer, 1));
                                        LOG.info().$("i=").$(i).$(", j=").$(k).$();
                                        Assert.assertEquals('A', Unsafe.getUnsafe().getByte(buffer));
                                    } finally {
                                        Unsafe.free(buffer, len, MemoryTag.NATIVE_DEFAULT);
                                    }
                                } finally {
                                    Net.close(fd);
                                }
                            }
                        } finally {
                            completedCount.incrementAndGet();
                            Net.freeSockAddr(sockAddr);
                            senderHalt.countDown();
                        }
                    }).start();
                }
                int receiveCount = 0;
                while (receiveCount < N * senderCount) {
                    long cursor = subSeq.next();
                    if (cursor < 0) {
                        if (cursor == -1 && completedCount.get() == senderCount) {
                            Assert.fail("Not all requests successful, test failed, see previous failures");
                            break;
                        }
                        Thread.yield();
                        continue;
                    }
                    boolean valid = queue.get(cursor).valid;
                    subSeq.done(cursor);
                    Assert.assertTrue(valid);
                    receiveCount++;
                }
            } catch (Throwable e) {
                e.printStackTrace();
                throw e;
            } finally {
                serverRunning.set(false);
                serverHaltLatch.await();
            }
        } catch (Throwable e) {
            e.printStackTrace();
            throw e;
        } finally {
            finished.set(true);
            senderHalt.await();
        }
        Assert.assertEquals(N * senderCount, requestsReceived.get());
    });
}

Example 9 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class ImportIODispatcherTest method testImportDesignatedTsFromSchema.

@Test
public void testImportDesignatedTsFromSchema() throws Exception {
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(1).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run(engine -> {
        setupSql(engine);
        final SOCountDownLatch waitForData = new SOCountDownLatch(1);
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (event == PoolListener.EV_RETURN && Chars.equals("syms", name)) {
                waitForData.countDown();
            }
        });
        new SendAndReceiveRequestBuilder().execute("POST /upload?name=syms&timestamp=ts1 HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "User-Agent: curl/7.64.0\r\n" + "Accept: */*\r\n" + "Content-Length: 437760673\r\n" + "Content-Type: multipart/form-data; boundary=------------------------27d997ca93d2689d\r\n" + "Expect: 100-continue\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"schema\"; filename=\"schema.json\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "[\r\n" + "  {\r\n" + "    \"name\": \"col1\",\r\n" + "    \"type\": \"SYMBOL\",\r\n" + "    \"index\": \"false\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"col2\",\r\n" + "    \"type\": \"SYMBOL\",\r\n" + "    \"index\": \"true\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"ts1\",\r\n" + "    \"type\": \"TIMESTAMP\",\r\n" + "    \"pattern\": \"yyyy-MM-dd HH:mm:ss\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"ts2\",\r\n" + "    \"type\": \"TIMESTAMP\",\r\n" + "    \"pattern\": \"yyyy-MM-dd HH:mm:ss\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"ts3\",\r\n" + "    \"type\": \"TIMESTAMP\",\r\n" + "    \"pattern\": \"yyyy-MM-dd HH:mm:ss\"\r\n" + "  }\r\n" + "]\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"data\"; filename=\"table2.csv\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "col1,col2,ts1,ts2,ts3\r\n" + "sym1,sym2,,2017-02-01 00:30:00,2017-02-01 00:30:01\r\n" + "\r\n" + "--------------------------27d997ca93d2689d--", "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0666\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|      Location:  |                                              syms  |        Pattern  | Locale  |      Errors  |\r\n" + "|   Partition by  |                                              NONE  |                 |         |              |\r\n" + "|      Timestamp  |                                               ts1  |                 |         |              |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|   Rows handled  |                                                 1  |                 |         |              |\r\n" + "|  Rows imported  |                                                 0  |                 |         |              |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|              0  |                                              col1  |                   SYMBOL  |           0  |\r\n" + "|              1  |                                              col2  |         (idx/256) SYMBOL  |           0  |\r\n" + "|              2  |                                               ts1  |                TIMESTAMP  |           1  |\r\n" + "|              3  |                                               ts2  |                TIMESTAMP  |           0  |\r\n" + "|              4  |                                               ts3  |                TIMESTAMP  |           0  |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "\r\n" + "00\r\n" + "\r\n");
        if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
            Assert.fail();
        }
        try (TableReader reader = new TableReader(engine.getConfiguration(), "syms")) {
            TableReaderMetadata meta = reader.getMetadata();
            Assert.assertEquals(5, meta.getColumnCount());
            Assert.assertEquals(2, meta.getTimestampIndex());
            Assert.assertEquals(ColumnType.SYMBOL, meta.getColumnType("col1"));
            Assert.assertFalse(meta.isColumnIndexed(0));
            Assert.assertEquals(ColumnType.SYMBOL, meta.getColumnType("col2"));
            Assert.assertTrue(meta.isColumnIndexed(1));
            Assert.assertEquals(ColumnType.TIMESTAMP, meta.getColumnType("ts1"));
            Assert.assertFalse(meta.isColumnIndexed(2));
            Assert.assertEquals(ColumnType.TIMESTAMP, meta.getColumnType("ts2"));
            Assert.assertFalse(meta.isColumnIndexed(3));
            Assert.assertEquals(ColumnType.TIMESTAMP, meta.getColumnType("ts3"));
            Assert.assertFalse(meta.isColumnIndexed(4));
        }
        compiler.close();
    });
}

Example 10 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LogFactoryTest method testRollingFileWriterBySize.

@Test
public void testRollingFileWriterBySize() throws Exception {
    String base = temp.getRoot().getAbsolutePath() + Files.SEPARATOR;
    String logFile = base + "mylog-${date:yyyy-MM-dd}.log";
    String expectedLogFile = base + "mylog-2015-05-03.log";
    final MicrosecondClock clock = new TestMicrosecondClock(TimestampFormatUtils.parseTimestamp("2015-05-03T10:35:00.000Z"), 1);
    try (Path path = new Path()) {
        // create rogue file that would be in a way of logger rolling existing files
        path.of(base);
        Assert.assertTrue(Files.touch(path.concat("mylog-2015-05-03.log.2").$()));
    }
    RingQueue<LogRecordSink> queue = new RingQueue<>(LogRecordSink::new, 1024, 1024, MemoryTag.NATIVE_DEFAULT);
    SPSequence pubSeq = new SPSequence(queue.getCycle());
    SCSequence subSeq = new SCSequence();
    pubSeq.then(subSeq).then(pubSeq);
    try (final LogRollingFileWriter writer = new LogRollingFileWriter(FilesFacadeImpl.INSTANCE, clock, queue, subSeq, LogLevel.LOG_LEVEL_INFO)) {
        writer.setLocation(logFile);
        writer.setRollSize("1m");
        writer.setBufferSize("64k");
        writer.bindProperties();
        AtomicBoolean running = new AtomicBoolean(true);
        SOCountDownLatch halted = new SOCountDownLatch();
        halted.setCount(1);
        new Thread(() -> {
            while (running.get()) {
                writer.runSerially();
            }
            // noinspection StatementWithEmptyBody
            while (writer.runSerially()) ;
            halted.countDown();
        }).start();
        // now publish
        int published = 0;
        int toPublish = 100_000;
        while (published < toPublish) {
            long cursor = pubSeq.next();
            if (cursor < 0) {
                LockSupport.parkNanos(1);
                continue;
            }
            final long available = pubSeq.available();
            while (cursor < available && published < toPublish) {
                LogRecordSink sink = queue.get(cursor++);
                sink.setLevel(LogLevel.LOG_LEVEL_INFO);
                sink.put("test");
                published++;
            }
            pubSeq.done(cursor - 1);
        }
        running.set(false);
        halted.await();
    }
    assertFileLength(expectedLogFile);
    assertFileLength(expectedLogFile + ".1");
}