use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.
the class ColumnIndexerJob method doRun.
protected boolean doRun(int workerId, long cursor) {
final ColumnIndexerTask queueItem = queue.get(cursor);
// copy values and release queue item
final ColumnIndexer indexer = queueItem.indexer;
final long lo = queueItem.lo;
final long hi = queueItem.hi;
final long indexSequence = queueItem.sequence;
final SOCountDownLatch latch = queueItem.countDownLatch;
subSeq.done(cursor);
// Using CAS allows main thread to steal only parts of its own job.
if (indexer.tryLock(indexSequence)) {
TableWriter.indexAndCountDown(indexer, lo, hi, latch);
return true;
}
// work from under nose of this worker.
return false;
}
use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.
the class CairoEngineTest method testNextTableId.
@Test
public void testNextTableId() {
try (CairoEngine engine = new CairoEngine(configuration);
CairoEngine engineB = new CairoEngine(configuration)) {
final LongList listA = new LongList();
final LongList listB = new LongList();
final CyclicBarrier startBarrier = new CyclicBarrier(2);
final SOCountDownLatch haltLatch = new SOCountDownLatch();
haltLatch.setCount(1);
final AtomicInteger errors = new AtomicInteger();
new Thread(() -> {
try {
startBarrier.await();
for (int i = 0; i < 100; i++) {
listA.add(engine.getNextTableId());
}
haltLatch.countDown();
} catch (InterruptedException | BrokenBarrierException e) {
e.printStackTrace();
errors.incrementAndGet();
}
}).start();
try {
startBarrier.await();
for (int i = 0; i < 100; i++) {
listB.add(engineB.getNextTableId());
}
} catch (InterruptedException | BrokenBarrierException e) {
e.printStackTrace();
errors.incrementAndGet();
}
haltLatch.await();
try (LongTreeSet set = new LongTreeSet(4 * 2048, Integer.MAX_VALUE)) {
// add both arrays to the set and asset that there are no duplicates
for (int i = 0, n = listA.size(); i < n; i++) {
Assert.assertTrue(set.put(listA.getQuick(i)));
}
for (int i = 0, n = listB.size(); i < n; i++) {
Assert.assertTrue(set.put(listB.getQuick(i)));
}
}
}
}
use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.
the class O3FailureTest method testOutOfFileHandles0.
private static void testOutOfFileHandles0(CairoEngine engine, SqlCompiler compiler, SqlExecutionContext executionContext) throws SqlException {
compiler.compile("create table x as (" + "select" + " rnd_str(5,16,2) i," + " rnd_str(5,16,2) sym," + " rnd_str(5,16,2) amt," + " rnd_str(5,16,2) timestamp," + " rnd_str(5,16,2) b," + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + " rnd_str(5,16,2) d," + " rnd_str(5,16,2) e," + " rnd_str(5,16,2) f," + " rnd_str(5,16,2) g," + " rnd_str(5,16,2) ik," + " rnd_str(5,16,2) j," + " timestamp_sequence(500000000000L,100000000L) ts," + " rnd_str(5,16,2) l," + " rnd_str(5,16,2) m," + " rnd_str(5,16,2) n," + " rnd_str(5,16,2) t," + " rnd_str(5,16,2) l256" + " from long_sequence(10000)" + ") timestamp (ts) partition by DAY", executionContext);
compiler.compile("create table x1 as (x) timestamp(ts) partition by DAY", executionContext);
compiler.compile("create table y as (" + "select" + " rnd_str(5,16,2) i," + " rnd_str(5,16,2) sym," + " rnd_str(5,16,2) amt," + " rnd_str(5,16,2) timestamp," + " rnd_str(5,16,2) b," + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + " rnd_str(5,16,2) d," + " rnd_str(5,16,2) e," + " rnd_str(5,16,2) f," + " rnd_str(5,16,2) g," + " rnd_str(5,16,2) ik," + " rnd_str(5,16,2) j," + " timestamp_sequence(500000080000L,79999631L) ts," + " rnd_str(5,16,2) l," + " rnd_str(5,16,2) m," + " rnd_str(5,16,2) n," + " rnd_str(5,16,2) t," + " rnd_str(5,16,2) l256" + " from long_sequence(10000)" + ") timestamp (ts) partition by DAY", executionContext);
compiler.compile("create table y1 as (y)", executionContext);
// create expected result sets
compiler.compile("create table z as (x union all y)", executionContext);
// create another compiler to be used by second pool
try (SqlCompiler compiler2 = new SqlCompiler(engine)) {
final CyclicBarrier barrier = new CyclicBarrier(2);
final SOCountDownLatch haltLatch = new SOCountDownLatch(2);
final AtomicInteger errorCount = new AtomicInteger();
// we have two pairs of tables (x,y) and (x1,y1)
WorkerPool pool1 = new WorkerPool(new WorkerPoolAwareConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1 };
}
@Override
public int getWorkerCount() {
return 1;
}
@Override
public boolean haltOnError() {
return false;
}
@Override
public boolean isEnabled() {
return true;
}
});
pool1.assign(new Job() {
private boolean toRun = true;
@Override
public boolean run(int workerId) {
if (toRun) {
try {
toRun = false;
barrier.await();
compiler.compile("insert into x select * from y", executionContext);
} catch (Throwable e) {
e.printStackTrace();
errorCount.incrementAndGet();
} finally {
haltLatch.countDown();
}
}
return false;
}
});
pool1.assignCleaner(Path.CLEANER);
final WorkerPool pool2 = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1 };
}
@Override
public int getWorkerCount() {
return 1;
}
@Override
public boolean haltOnError() {
return false;
}
});
pool2.assign(new Job() {
private boolean toRun = true;
@Override
public boolean run(int workerId) {
if (toRun) {
try {
toRun = false;
barrier.await();
compiler2.compile("insert into x1 select * from y1", executionContext);
} catch (Throwable e) {
e.printStackTrace();
errorCount.incrementAndGet();
} finally {
haltLatch.countDown();
}
}
return false;
}
});
pool2.assignCleaner(Path.CLEANER);
pool1.start(null);
pool2.start(null);
haltLatch.await();
pool1.halt();
pool2.halt();
Assert.assertTrue(errorCount.get() > 0);
}
}
use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.
the class SqlCodeGeneratorTest method testVectorSumAvgDoubleRndColumnWithNullsParallel.
@Test
public void testVectorSumAvgDoubleRndColumnWithNullsParallel() throws Exception {
final AtomicBoolean running = new AtomicBoolean(true);
final SOCountDownLatch haltLatch = new SOCountDownLatch(1);
final GroupByJob job = new GroupByJob(engine.getMessageBus());
new Thread(() -> {
while (running.get()) {
job.run(0);
}
haltLatch.countDown();
}).start();
try {
assertQuery("avg\tsum\n" + "0.50035043\t834470.437288\n", "select round(avg(c), 9) avg, round(sum(c), 6) sum from x", "create table x as (select rnd_int(0,100,2) a, rnd_double(2) b, rnd_double(2) c, rnd_int() d from long_sequence(2000000))", null, false, true, true);
} finally {
running.set(false);
haltLatch.await();
}
}
use of io.questdb.mp.SOCountDownLatch in project questdb by bluestreak01.
the class IODispatcherTest method testSendHttpGetAndSimpleResponse.
@Test
public void testSendHttpGetAndSimpleResponse() throws Exception {
LOG.info().$("started testSendHttpGetAndSimpleResponse").$();
final String request = "GET /status?x=1&a=%26b&c&d=x HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
// the difference between request and expected is url encoding (and ':' padding, which can easily be fixed)
final String expected = "GET /status?x=1&a=&b&c&d=x HTTP/1.1\r\n" + "host:localhost:9000\r\n" + "connection:keep-alive\r\n" + "cache-control:max-age=0\r\n" + "accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "user-agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "accept-encoding:gzip,deflate,sdch\r\n" + "accept-language:en-US,en;q=0.8\r\n" + "cookie:textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
final String expectedResponse = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "04\r\n" + "OK\r\n" + "\r\n" + "00\r\n" + "\r\n";
assertMemoryLeak(() -> {
HttpServerConfiguration httpServerConfiguration = new DefaultHttpServerConfiguration(new DefaultHttpContextConfiguration() {
@Override
public MillisecondClock getClock() {
return () -> 0;
}
});
SOCountDownLatch connectLatch = new SOCountDownLatch(1);
SOCountDownLatch contextClosedLatch = new SOCountDownLatch(1);
AtomicInteger closeCount = new AtomicInteger(0);
try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration(), new IOContextFactory<HttpConnectionContext>() {
@Override
public HttpConnectionContext newInstance(long fd, IODispatcher<HttpConnectionContext> dispatcher1) {
connectLatch.countDown();
return new HttpConnectionContext(httpServerConfiguration.getHttpContextConfiguration()) {
@Override
public void close() {
// context is closed
if (closeCount.incrementAndGet() == 1) {
super.close();
contextClosedLatch.countDown();
}
}
}.of(fd, dispatcher1);
}
})) {
StringSink sink = new StringSink();
final HttpRequestProcessorSelector selector = new HttpRequestProcessorSelector() {
@Override
public HttpRequestProcessor select(CharSequence url) {
return null;
}
@Override
public HttpRequestProcessor getDefaultProcessor() {
return new HttpRequestProcessor() {
@Override
public void onHeadersReady(HttpConnectionContext context) {
HttpRequestHeader headers = context.getRequestHeader();
sink.put(headers.getMethodLine());
sink.put("\r\n");
ObjList<CharSequence> headerNames = headers.getHeaderNames();
for (int i = 0, n = headerNames.size(); i < n; i++) {
sink.put(headerNames.getQuick(i)).put(':');
sink.put(headers.getHeader(headerNames.getQuick(i)));
sink.put("\r\n");
}
sink.put("\r\n");
}
@Override
public void onRequestComplete(HttpConnectionContext context) throws PeerDisconnectedException, PeerIsSlowToReadException {
context.simpleResponse().sendStatusWithDefaultMessage(200);
}
};
}
@Override
public void close() {
}
};
AtomicBoolean serverRunning = new AtomicBoolean(true);
SOCountDownLatch serverHaltLatch = new SOCountDownLatch(1);
new Thread(() -> {
while (serverRunning.get()) {
dispatcher.run(0);
dispatcher.processIOQueue((operation, context) -> context.handleClientOperation(operation, selector, EmptyRescheduleContext));
}
serverHaltLatch.countDown();
}).start();
long fd = Net.socketTcp(true);
try {
long sockAddr = Net.sockaddr("127.0.0.1", 9001);
try {
TestUtils.assertConnect(fd, sockAddr);
connectLatch.await();
int len = request.length();
long buffer = TestUtils.toMemory(request);
try {
Assert.assertEquals(len, Net.send(fd, buffer, len));
// read response we expect
StringSink sink2 = new StringSink();
final int expectedLen = expectedResponse.length();
int read = 0;
while (read < expectedLen) {
int n = Net.recv(fd, buffer, len);
Assert.assertTrue(n > 0);
for (int i = 0; i < n; i++) {
sink2.put((char) Unsafe.getUnsafe().getByte(buffer + i));
}
// copy response bytes to sink
read += n;
}
TestUtils.assertEquals(expectedResponse, sink2);
} finally {
Unsafe.free(buffer, len, MemoryTag.NATIVE_DEFAULT);
}
Assert.assertEquals(0, Net.close(fd));
LOG.info().$("closed [fd=").$(fd).$(']').$();
fd = -1;
contextClosedLatch.await();
serverRunning.set(false);
serverHaltLatch.await();
Assert.assertEquals(0, dispatcher.getConnectionCount());
TestUtils.assertEquals(expected, sink);
} finally {
Net.freeSockAddr(sockAddr);
}
} finally {
if (fd != -1) {
Net.close(fd);
}
}
Assert.assertEquals(1, closeCount.get());
}
});
}
Aggregations