Examples with SOCountDownLatch io.questdb.mp.SOCountDownLatch used on opensource projects

Example 11 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LineTcpReceiverTest method test.

private void test(String authKeyId, PrivateKey authPrivateKey, int msgBufferSize, final int nRows, boolean expectDisconnect) throws Exception {
    this.authKeyId = authKeyId;
    this.msgBufferSize = msgBufferSize;
    assertMemoryLeak(() -> {
        final String[] locations = { "x london", "paris", "rome" };
        final CharSequenceHashSet tables = new CharSequenceHashSet();
        tables.add("weather1");
        tables.add("weather2");
        tables.add("weather3");
        SOCountDownLatch tablesCreated = new SOCountDownLatch();
        tablesCreated.setCount(tables.size());
        final Rnd rand = new Rnd();
        final StringBuilder[] expectedSbs = new StringBuilder[tables.size()];
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN) {
                if (tables.contains(name)) {
                    tablesCreated.countDown();
                }
            }
        });
        minIdleMsBeforeWriterRelease = 100;
        try (LineTcpReceiver ignored = LineTcpReceiver.create(lineConfiguration, sharedWorkerPool, LOG, engine)) {
            long startEpochMs = System.currentTimeMillis();
            sharedWorkerPool.assignCleaner(Path.CLEANER);
            sharedWorkerPool.start(LOG);
            try {
                final AbstractLineSender[] senders = new AbstractLineSender[tables.size()];
                for (int n = 0; n < senders.length; n++) {
                    if (null != authKeyId) {
                        AuthenticatedLineTcpSender sender = new AuthenticatedLineTcpSender(authKeyId, authPrivateKey, Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                        sender.authenticate();
                        senders[n] = sender;
                    } else {
                        senders[n] = new LineTcpSender(Net.parseIPv4("127.0.0.1"), bindPort, 4096);
                    }
                    StringBuilder sb = new StringBuilder((nRows + 1) * lineConfiguration.getMaxMeasurementSize());
                    sb.append("location\ttemp\ttimestamp\n");
                    expectedSbs[n] = sb;
                }
                long ts = Os.currentTimeMicros();
                StringSink tsSink = new StringSink();
                for (int nRow = 0; nRow < nRows; nRow++) {
                    int nTable = nRow < tables.size() ? nRow : rand.nextInt(tables.size());
                    AbstractLineSender sender = senders[nTable];
                    StringBuilder sb = expectedSbs[nTable];
                    CharSequence tableName = tables.get(nTable);
                    sender.metric(tableName);
                    String location = locations[rand.nextInt(locations.length)];
                    sb.append(location);
                    sb.append('\t');
                    sender.tag("location", location);
                    int temp = rand.nextInt(100);
                    sb.append(temp);
                    sb.append('\t');
                    sender.field("temp", temp);
                    tsSink.clear();
                    TimestampFormatUtils.appendDateTimeUSec(tsSink, ts);
                    sb.append(tsSink);
                    sb.append('\n');
                    sender.$(ts * 1000);
                    sender.flush();
                    if (expectDisconnect) {
                        // To prevent all data being buffered before the expected disconnect slow sending
                        Os.sleep(100);
                    }
                    ts += rand.nextInt(1000);
                }
                for (int n = 0; n < senders.length; n++) {
                    AbstractLineSender sender = senders[n];
                    sender.close();
                }
                Assert.assertFalse(expectDisconnect);
                boolean ready = tablesCreated.await(TimeUnit.MINUTES.toNanos(1));
                if (!ready) {
                    throw new IllegalStateException("Timeout waiting for tables to be created");
                }
                int nRowsWritten;
                do {
                    nRowsWritten = 0;
                    long timeTakenMs = System.currentTimeMillis() - startEpochMs;
                    if (timeTakenMs > TEST_TIMEOUT_IN_MS) {
                        LOG.error().$("after ").$(timeTakenMs).$("ms tables only had ").$(nRowsWritten).$(" rows out of ").$(nRows).$();
                        break;
                    }
                    Thread.yield();
                    for (int n = 0; n < tables.size(); n++) {
                        CharSequence tableName = tables.get(n);
                        while (true) {
                            try (TableReader reader = engine.getReader(AllowAllCairoSecurityContext.INSTANCE, tableName)) {
                                TableReaderRecordCursor cursor = reader.getCursor();
                                while (cursor.hasNext()) {
                                    nRowsWritten++;
                                }
                                break;
                            } catch (EntryLockedException ex) {
                                LOG.info().$("retrying read for ").$(tableName).$();
                                LockSupport.parkNanos(1);
                            }
                        }
                    }
                } while (nRowsWritten < nRows);
                LOG.info().$(nRowsWritten).$(" rows written").$();
            } finally {
                sharedWorkerPool.halt();
            }
        } finally {
            engine.setPoolListener(null);
        }
        for (int n = 0; n < tables.size(); n++) {
            CharSequence tableName = tables.get(n);
            LOG.info().$("checking table ").$(tableName).$();
            assertTable(expectedSbs[n], tableName);
        }
    });
}

Example 12 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LineUdpInsertTest method assertType.

protected static void assertType(String tableName, String targetColumnName, int columnType, String expected, Consumer<AbstractLineSender> senderConsumer, String... expectedExtraStringColumns) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (CairoEngine engine = new CairoEngine(configuration)) {
            final SOCountDownLatch waitForData = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (event == PoolListener.EV_RETURN && Chars.equals(tableName, name)) {
                    waitForData.countDown();
                }
            });
            try (AbstractLineProtoUdpReceiver receiver = createLineProtoReceiver(engine)) {
                if (columnType != ColumnType.UNDEFINED) {
                    try (TableModel model = new TableModel(configuration, tableName, PartitionBy.NONE)) {
                        CairoTestUtils.create(model.col(targetColumnName, columnType).timestamp());
                    }
                }
                receiver.start();
                try (AbstractLineSender sender = createLineProtoSender()) {
                    senderConsumer.accept(sender);
                    sender.flush();
                }
                // allow reader to hit the readout
                Os.sleep(250L);
            }
            if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
                Assert.fail();
            }
            assertReader(tableName, expected, expectedExtraStringColumns);
        }
    });
}

Example 13 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LineUdpParserSupportTest method testColumnType.

private void testColumnType(int columnType, String expected, Consumer<AbstractLineSender> senderConsumer) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (CairoEngine engine = new CairoEngine(configuration)) {
            final SOCountDownLatch waitForData = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (event == PoolListener.EV_RETURN && Chars.equals(tableName, name)) {
                    waitForData.countDown();
                }
            });
            try (AbstractLineProtoUdpReceiver receiver = createLineProtoReceiver(engine)) {
                try (TableModel model = new TableModel(configuration, tableName, PartitionBy.NONE)) {
                    CairoTestUtils.create(model.col(targetColumnName, columnType).col(locationColumnName, ColumnType.getGeoHashTypeWithBits(30)).timestamp());
                }
                receiver.start();
                try (AbstractLineSender sender = createLineProtoSender()) {
                    senderConsumer.accept(sender);
                    sender.flush();
                }
                Os.sleep(250L);
            }
            if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
                Assert.fail();
            }
        }
        assertReader(tableName, expected);
    });
}

Example 14 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LineTcpO3Test method test.

private void test(String ilpResourceName) throws Exception {
    assertMemoryLeak(() -> {
        long clientFd = Net.socketTcp(true);
        Assert.assertTrue(clientFd >= 0);
        long ilpSockAddr = Net.sockaddr(Net.parseIPv4("127.0.0.1"), lineConfiguration.getNetDispatcherConfiguration().getBindPort());
        WorkerPool sharedWorkerPool = new WorkerPool(sharedWorkerPoolConfiguration);
        try (LineTcpReceiver ignored = LineTcpReceiver.create(lineConfiguration, sharedWorkerPool, LOG, engine);
            SqlCompiler compiler = new SqlCompiler(engine);
            SqlExecutionContext sqlExecutionContext = new SqlExecutionContextImpl(engine, 1)) {
            SOCountDownLatch haltLatch = new SOCountDownLatch(1);
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
                if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN && Chars.equals(name, "cpu")) {
                    haltLatch.countDown();
                }
            });
            sharedWorkerPool.assignCleaner(Path.CLEANER);
            sharedWorkerPool.start(LOG);
            TestUtils.assertConnect(clientFd, ilpSockAddr);
            readGzResource(ilpResourceName);
            Net.send(clientFd, resourceAddress, resourceSize);
            Unsafe.free(resourceAddress, resourceSize, MemoryTag.NATIVE_DEFAULT);
            haltLatch.await();
            TestUtils.printSql(compiler, sqlExecutionContext, "select * from " + "cpu", sink);
            readGzResource("selectAll1");
            DirectUnboundedByteSink expectedSink = new DirectUnboundedByteSink(resourceAddress);
            expectedSink.clear(resourceSize);
            TestUtils.assertEquals(expectedSink.toString(), sink);
            Unsafe.free(resourceAddress, resourceSize, MemoryTag.NATIVE_DEFAULT);
        } finally {
            engine.setPoolListener(null);
            Net.close(clientFd);
            Net.freeSockAddr(ilpSockAddr);
            sharedWorkerPool.halt();
        }
    });
}

Example 15 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class IODispatcherTest method testJsonQueryWithInterruption.

@Test
public void testJsonQueryWithInterruption() throws Exception {
    assertMemoryLeak(() -> {
        final NetworkFacade nf = NetworkFacadeImpl.INSTANCE;
        final String baseDir = temp.getRoot().getAbsolutePath();
        final int tableRowCount = 300_000;
        SOCountDownLatch peerDisconnectLatch = new SOCountDownLatch(1);
        DefaultHttpServerConfiguration httpConfiguration = new HttpServerConfigurationBuilder().withNetwork(nf).withBaseDir(baseDir).withSendBufferSize(256).withDumpingTraffic(false).withAllowDeflateBeforeSend(false).withServerKeepAlive(true).withHttpProtocolVersion("HTTP/1.1 ").withOnPeerDisconnect(peerDisconnectLatch::countDown).build();
        QueryCache.configure(httpConfiguration);
        final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        try (CairoEngine engine = new CairoEngine(new DefaultCairoConfiguration(baseDir));
            HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new StaticContentProcessor(httpConfiguration);
                }

                @Override
                public String getUrl() {
                    return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
                }
            });
            httpServer.bind(new HttpRequestProcessorFactory() {

                @Override
                public HttpRequestProcessor newInstance() {
                    return new JsonQueryProcessor(httpConfiguration.getJsonQueryProcessorConfiguration(), engine, workerPool.getWorkerCount(), metrics);
                }

                @Override
                public String getUrl() {
                    return "/query";
                }
            });
            final int minClientReceivedBytesBeforeDisconnect = 180;
            final AtomicLong refClientFd = new AtomicLong(-1);
            HttpClientStateListener clientStateListener = new HttpClientStateListener() {

                private int nBytesReceived = 0;

                @Override
                public void onClosed() {
                }

                @Override
                public void onReceived(int nBytes) {
                    LOG.info().$("Client received ").$(nBytes).$(" bytes").$();
                    nBytesReceived += nBytes;
                    if (nBytesReceived >= minClientReceivedBytesBeforeDisconnect) {
                        long fd = refClientFd.get();
                        if (fd != -1) {
                            refClientFd.set(-1);
                            nf.close(fd);
                        }
                    }
                }
            };
            workerPool.start(LOG);
            try {
                // create table with all column types
                CairoTestUtils.createTestTable(engine.getConfiguration(), tableRowCount, new Rnd(), new TestRecord.ArrayBinarySequence());
                // send multipart request to server
                final String request = "GET /query?query=select+distinct+a+from+x+where+test_latched_counter() HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Upgrade-Insecure-Requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n";
                long fd = nf.socketTcp(true);
                try {
                    long sockAddr = nf.sockaddr("127.0.0.1", 9001);
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        Assert.assertEquals(0, nf.setTcpNoDelay(fd, true));
                        refClientFd.set(fd);
                        nf.configureNonBlocking(fd);
                        long bufLen = request.length();
                        long ptr = Unsafe.malloc(bufLen, MemoryTag.NATIVE_DEFAULT);
                        try {
                            new SendAndReceiveRequestBuilder().withNetworkFacade(nf).withPauseBetweenSendAndReceive(0).withPrintOnly(false).withExpectDisconnect(true).executeUntilDisconnect(request, fd, 200, ptr, clientStateListener);
                        } finally {
                            Unsafe.free(ptr, bufLen, MemoryTag.NATIVE_DEFAULT);
                        }
                    } finally {
                        nf.freeSockAddr(sockAddr);
                    }
                } finally {
                    LOG.info().$("Closing client connection").$();
                }
                peerDisconnectLatch.await();
                // depending on how quick the CI hardware is we may end up processing different
                // number of rows before query is interrupted
                Assert.assertTrue(tableRowCount > TestLatchedCounterFunctionFactory.getCount());
            } finally {
                workerPool.halt();
            }
        }
    });
}