Examples with SOCountDownLatch io.questdb.mp.SOCountDownLatch used on opensource projects

Example 21 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class ReaderPoolTest method testLockBusyReader.

@Test
public void testLockBusyReader() throws Exception {
    assertWithPool(new PoolAwareCode() {

        @Override
        public void run(ReaderPool pool) throws Exception {
            try (TableModel model = new TableModel(configuration, "x", PartitionBy.NONE).col("ts", ColumnType.DATE)) {
                CairoTestUtils.create(model);
            }
            final int N = 100_000;
            for (int i = 0; i < N; i++) {
                testLockBusyReaderRollTheDice(pool);
            }
        }

        private void testLockBusyReaderRollTheDice(ReaderPool pool) throws BrokenBarrierException, InterruptedException {
            final CyclicBarrier start = new CyclicBarrier(2);
            final SOCountDownLatch halt = new SOCountDownLatch(1);
            final AtomicReference<TableReader> ref = new AtomicReference<>();
            new Thread(() -> {
                try {
                    // start together with main thread
                    start.await();
                    // try to get reader from pool
                    ref.set(pool.get("x"));
                } catch (Throwable ignored) {
                } finally {
                    // the end
                    halt.countDown();
                }
            }).start();
            // start together with the thread
            start.await();
            // get a lock
            boolean couldLock = pool.lock("x");
            // wait until thread stops
            halt.await();
            // assert
            if (couldLock) {
                pool.unlock("x");
                Assert.assertNull(ref.get());
            } else {
                TableReader reader = ref.get();
                Assert.assertNotNull(reader);
                reader.close();
            }
        }
    });
}

Example 22 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class ImportIODispatcherTest method testImportSymbolIndexedFromSchema.

@Test
public void testImportSymbolIndexedFromSchema() throws Exception {
    new HttpQueryTestBuilder().withTempFolder(temp).withWorkerCount(1).withHttpServerConfigBuilder(new HttpServerConfigurationBuilder()).withTelemetry(false).run(engine -> {
        setupSql(engine);
        final SOCountDownLatch waitForData = new SOCountDownLatch(1);
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (event == PoolListener.EV_RETURN && Chars.equals("syms", name)) {
                waitForData.countDown();
            }
        });
        new SendAndReceiveRequestBuilder().execute("POST /upload?name=syms&timestamp=ts HTTP/1.1\r\n" + "Host: localhost:9001\r\n" + "User-Agent: curl/7.64.0\r\n" + "Accept: */*\r\n" + "Content-Length: 437760673\r\n" + "Content-Type: multipart/form-data; boundary=------------------------27d997ca93d2689d\r\n" + "Expect: 100-continue\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"schema\"; filename=\"schema.json\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "[\r\n" + "  {\r\n" + "    \"name\": \"col1\",\r\n" + "    \"type\": \"SYMBOL\",\r\n" + "    \"index\": \"true\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"col2\",\r\n" + "    \"type\": \"SYMBOL\",\r\n" + "    \"index\": \"false\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"col3\",\r\n" + "    \"type\": \"SYMBOL\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"col4\",\r\n" + "    \"type\": \"STRING\",\r\n" + "    \"index\": \"true\"\r\n" + "  },\r\n" + "  {\r\n" + "    \"name\": \"ts\",\r\n" + "    \"type\": \"TIMESTAMP\",\r\n" + "    \"pattern\": \"yyyy-MM-dd HH:mm:ss\"\r\n" + "  }\r\n" + "]\r\n" + "\r\n" + "--------------------------27d997ca93d2689d\r\n" + "Content-Disposition: form-data; name=\"data\"; filename=\"table2.csv\"\r\n" + "Content-Type: application/octet-stream\r\n" + "\r\n" + "col1,col2,col3,col4,ts\r\n" + "sym1,sym2,,string here,2017-02-01 00:30:00\r\n" + "\r\n" + "--------------------------27d997ca93d2689d--", "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0666\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|      Location:  |                                              syms  |        Pattern  | Locale  |      Errors  |\r\n" + "|   Partition by  |                                              NONE  |                 |         |              |\r\n" + "|      Timestamp  |                                                ts  |                 |         |              |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|   Rows handled  |                                                 1  |                 |         |              |\r\n" + "|  Rows imported  |                                                 1  |                 |         |              |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "|              0  |                                              col1  |         (idx/256) SYMBOL  |           0  |\r\n" + "|              1  |                                              col2  |                   SYMBOL  |           0  |\r\n" + "|              2  |                                              col3  |                   SYMBOL  |           0  |\r\n" + "|              3  |                                              col4  |                   STRING  |           0  |\r\n" + "|              4  |                                                ts  |                TIMESTAMP  |           0  |\r\n" + "+-----------------------------------------------------------------------------------------------------------------+\r\n" + "\r\n" + "00\r\n" + "\r\n");
        if (!waitForData.await(TimeUnit.SECONDS.toNanos(30L))) {
            Assert.fail();
        }
        try (TableReader reader = new TableReader(engine.getConfiguration(), "syms")) {
            TableReaderMetadata meta = reader.getMetadata();
            Assert.assertEquals(5, meta.getColumnCount());
            Assert.assertEquals(ColumnType.SYMBOL, meta.getColumnType("col1"));
            Assert.assertTrue(meta.isColumnIndexed(0));
            Assert.assertEquals(ColumnType.SYMBOL, meta.getColumnType("col2"));
            Assert.assertFalse(meta.isColumnIndexed(1));
            Assert.assertEquals(ColumnType.SYMBOL, meta.getColumnType("col3"));
            Assert.assertFalse(meta.isColumnIndexed(2));
            Assert.assertEquals(ColumnType.STRING, meta.getColumnType("col4"));
            Assert.assertFalse(meta.isColumnIndexed(3));
            Assert.assertEquals(ColumnType.TIMESTAMP, meta.getColumnType("ts"));
            Assert.assertFalse(meta.isColumnIndexed(4));
        }
        compiler.close();
    });
}

Example 23 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class LineTcpReceiverTest method testSomeWritersReleased.

@Test
public void testSomeWritersReleased() throws Exception {
    runInContext((receiver) -> {
        String lineData = "weather,location=us-midwest temperature=85 1465839830102300200\n" + "weather,location=us-eastcoast temperature=89 1465839830102400200\n" + "weather,location=us-westcost temperature=82 1465839830102500200\n";
        int iterations = 8;
        int threadCount = 8;
        CharSequenceObjHashMap<SOUnboundedCountDownLatch> tableIndex = new CharSequenceObjHashMap<>();
        tableIndex.put("weather", new SOUnboundedCountDownLatch());
        for (int i = 1; i < threadCount; i++) {
            tableIndex.put("weather" + i, new SOUnboundedCountDownLatch());
        }
        // One engine hook for all writers
        engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            if (factoryType == PoolListener.SRC_WRITER && event == PoolListener.EV_RETURN) {
                tableIndex.get(name).countDown();
            }
        });
        try {
            sendAndWait(receiver, lineData, tableIndex, 1);
            SOCountDownLatch threadPushFinished = new SOCountDownLatch(threadCount - 1);
            for (int i = 1; i < threadCount; i++) {
                final String threadTable = "weather" + i;
                final String lineDataThread = lineData.replace("weather", threadTable);
                sendNoWait(receiver, threadTable, lineDataThread);
                new Thread(() -> {
                    try {
                        for (int n = 0; n < iterations; n++) {
                            Os.sleep(minIdleMsBeforeWriterRelease - 50);
                            send(receiver, lineDataThread, threadTable, WAIT_NO_WAIT);
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    } finally {
                        threadPushFinished.countDown();
                    }
                }).start();
            }
            sendAndWait(receiver, lineData, tableIndex, 2);
            try (TableWriter w = engine.getWriter(AllowAllCairoSecurityContext.INSTANCE, "weather", "testing")) {
                w.truncate();
            }
            sendAndWait(receiver, lineData, tableIndex, 4);
            String header = "location\ttemperature\ttimestamp\n";
            String[] lines = { "us-midwest\t85.0\t2016-06-13T17:43:50.102300Z\n", "us-eastcoast\t89.0\t2016-06-13T17:43:50.102400Z\n", "us-westcost\t82.0\t2016-06-13T17:43:50.102500Z\n" };
            String expected = header + lines[0] + lines[1] + lines[2];
            assertTable(expected, "weather");
            // Concatenate iterations + 1 of identical insert results
            // to assert against weather 1-8 tables
            StringBuilder expectedSB = new StringBuilder(header);
            for (int l = 0; l < lines.length; l++) {
                expectedSB.append(Chars.repeat(lines[l], iterations + 1));
            }
            // Wait async ILP send threads to finish.
            threadPushFinished.await();
            for (int i = 1; i < threadCount; i++) {
                // Wait writer to be released and check.
                String tableName = "weather" + i;
                try {
                    try {
                        assertTable(expectedSB, tableName);
                    } catch (AssertionError e) {
                        int releasedCount = -tableIndex.get(tableName).getCount();
                        // Wait one more writer release before re-trying to compare
                        wait(tableIndex.get(tableName), releasedCount + 1, minIdleMsBeforeWriterRelease);
                        assertTable(expectedSB, tableName);
                    }
                } catch (Throwable err) {
                    LOG.error().$("Error '").$(err.getMessage()).$("' comparing table: ").$(tableName).$();
                    throw err;
                }
            }
        } finally {
            // Clean engine hook
            engine.setPoolListener((factoryType, thread, name, event, segment, position) -> {
            });
        }
    });
}

Example 24 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class SymbolCacheTest method testConcurrency.

@Test
public void testConcurrency() throws Exception {
    assertMemoryLeak(() -> {
        final Rnd rndCache = new Rnd();
        final int N = 1_000_000;
        long ts = TimestampFormatUtils.parseTimestamp("2020-09-10T20:00:00.000000Z");
        final long incrementUs = 10000;
        final String constValue = "hello";
        compiler.compile("create table x(a symbol, c int, b symbol capacity 10000000, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext);
        try (SymbolCache symbolCache = new SymbolCache(new DefaultLineTcpReceiverConfiguration());
            Path path = new Path()) {
            path.of(configuration.getRoot()).concat("x");
            symbolCache.of(configuration, path, "b", 1);
            final CyclicBarrier barrier = new CyclicBarrier(2);
            final SOCountDownLatch haltLatch = new SOCountDownLatch(1);
            final AtomicBoolean cacheInError = new AtomicBoolean(false);
            RingQueue<Holder> wheel = new RingQueue<Holder>(Holder::new, 256);
            SPSequence pubSeq = new SPSequence(wheel.getCycle());
            SCSequence subSeq = new SCSequence();
            pubSeq.then(subSeq).then(pubSeq);
            new Thread(() -> {
                try {
                    barrier.await();
                    for (int i = 0; i < N; i++) {
                        // All keys should not be found, but we keep looking them up because
                        // we pretend we don't know this upfront. The aim is to cause
                        // race condition between lookup and table writer
                        final CharSequence value2 = rndCache.nextString(5);
                        symbolCache.getSymbolKey(constValue);
                        symbolCache.getSymbolKey(value2);
                        final long cursor = pubSeq.nextBully();
                        final Holder h = wheel.get(cursor);
                        // publish the value2 to the table writer
                        h.value1 = constValue;
                        h.value2 = Chars.toString(value2);
                        pubSeq.done(cursor);
                    }
                } catch (Throwable e) {
                    cacheInError.set(true);
                    e.printStackTrace();
                } finally {
                    haltLatch.countDown();
                }
            }).start();
            try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "x", "test")) {
                barrier.await();
                OUT: for (int i = 0; i < N; i++) {
                    long cursor;
                    while (true) {
                        cursor = subSeq.next();
                        if (cursor < 0) {
                            // due to random generator producing duplicate strings
                            if (haltLatch.getCount() < 1) {
                                break OUT;
                            }
                        } else {
                            break;
                        }
                    }
                    Holder h = wheel.get(cursor);
                    TableWriter.Row r = w.newRow(ts);
                    r.putSym(0, h.value1);
                    r.putInt(1, 0);
                    r.putSym(2, h.value2);
                    r.append();
                    subSeq.done(cursor);
                    if (i % 256 == 0) {
                        w.commit();
                    }
                    ts += incrementUs;
                }
                w.commit();
            } finally {
                haltLatch.await();
            }
            Assert.assertFalse(cacheInError.get());
        }
        compiler.compile("drop table x", sqlExecutionContext);
    });
}

Example 25 with SOCountDownLatch

use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.

the class O3Test method testTwoTablesCompeteForBuffer0.

private static void testTwoTablesCompeteForBuffer0(CairoEngine engine, SqlCompiler compiler, SqlExecutionContext executionContext) throws SqlException {
    compiler.compile("create table x as (" + "select" + " rnd_str(5,16,10) i," + " rnd_str(5,16,10) sym," + " rnd_str(5,16,10) amt," + " rnd_str(5,16,10) timestamp," + " rnd_str(5,16,10) b," + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + " rnd_str(5,16,10) d," + " rnd_str(5,16,10) e," + " rnd_str(5,16,10) f," + " rnd_str(5,16,10) g," + " rnd_str(5,16,10) ik," + " rnd_str(5,16,10) j," + " timestamp_sequence(500000000000L,100000000L) ts," + " rnd_str(5,16,10) l," + " rnd_str(5,16,10) m," + " rnd_str(5,16,10) n," + " rnd_str(5,16,10) t," + " rnd_str(5,16,10) l256" + " from long_sequence(10000)" + ") timestamp (ts) partition by DAY", executionContext);
    compiler.compile("create table x1 as (x) timestamp(ts) partition by DAY", executionContext);
    compiler.compile("create table y as (" + "select" + " rnd_str(5,16,10) i," + " rnd_str(5,16,10) sym," + " rnd_str(5,16,10) amt," + " rnd_str(5,16,10) timestamp," + " rnd_str(5,16,10) b," + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + " rnd_str(5,16,10) d," + " rnd_str(5,16,10) e," + " rnd_str(5,16,10) f," + " rnd_str(5,16,10) g," + " rnd_str(5,16,10) ik," + " rnd_str(5,16,10) j," + " timestamp_sequence(500000080000L,79999631L) ts," + " rnd_str(5,16,10) l," + " rnd_str(5,16,10) m," + " rnd_str(5,16,10) n," + " rnd_str(5,16,10) t," + " rnd_str(5,16,10) l256" + " from long_sequence(10000)" + ") timestamp (ts) partition by DAY", executionContext);
    compiler.compile("create table y1 as (y)", executionContext);
    // create expected result sets
    compiler.compile("create table z as (x union all y)", executionContext);
    // create another compiler to be used by second pool
    try (SqlCompiler compiler2 = new SqlCompiler(engine)) {
        final CyclicBarrier barrier = new CyclicBarrier(2);
        final SOCountDownLatch haltLatch = new SOCountDownLatch(2);
        final AtomicInteger errorCount = new AtomicInteger();
        // we have two pairs of tables (x,y) and (x1,y1)
        WorkerPool pool1 = new WorkerPool(new WorkerPoolAwareConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }

            @Override
            public boolean isEnabled() {
                return true;
            }
        });
        pool1.assign(new Job() {

            private boolean toRun = true;

            @Override
            public boolean run(int workerId) {
                if (toRun) {
                    try {
                        toRun = false;
                        barrier.await();
                        compiler.compile("insert into x select * from y", executionContext);
                    } catch (Throwable e) {
                        e.printStackTrace();
                        errorCount.incrementAndGet();
                    } finally {
                        haltLatch.countDown();
                    }
                }
                return false;
            }
        });
        pool1.assignCleaner(Path.CLEANER);
        final WorkerPool pool2 = new WorkerPool(new WorkerPoolConfiguration() {

            @Override
            public int[] getWorkerAffinity() {
                return new int[] { -1 };
            }

            @Override
            public int getWorkerCount() {
                return 1;
            }

            @Override
            public boolean haltOnError() {
                return false;
            }
        });
        pool2.assign(new Job() {

            private boolean toRun = true;

            @Override
            public boolean run(int workerId) {
                if (toRun) {
                    try {
                        toRun = false;
                        barrier.await();
                        compiler2.compile("insert into x1 select * from y1", executionContext);
                    } catch (Throwable e) {
                        e.printStackTrace();
                        errorCount.incrementAndGet();
                    } finally {
                        haltLatch.countDown();
                    }
                }
                return false;
            }
        });
        pool2.assignCleaner(Path.CLEANER);
        pool1.start(null);
        pool2.start(null);
        haltLatch.await();
        pool1.halt();
        pool2.halt();
        Assert.assertEquals(0, errorCount.get());
        TestUtils.assertSqlCursors(compiler, executionContext, "z order by ts", "x", LOG);
        TestUtils.assertSqlCursors(compiler, executionContext, "z order by ts", "x1", LOG);
    }
}