Search in sources :

Example 6 with IODispatcher

use of io.questdb.network.IODispatcher in project questdb by bluestreak01.

the class IODispatcherTest method testSendHttpGet.

@Test
public void testSendHttpGet() throws Exception {
    LOG.info().$("started testSendHttpGet").$();
    final String request = "GET /status?x=1&a=%26b&c&d=x HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
    // the difference between request and expected is url encoding (and ':' padding, which can easily be fixed)
    final String expected = "GET /status?x=1&a=&b&c&d=x HTTP/1.1\r\n" + "host:localhost:9000\r\n" + "connection:keep-alive\r\n" + "cache-control:max-age=0\r\n" + "accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "user-agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "accept-encoding:gzip,deflate,sdch\r\n" + "accept-language:en-US,en;q=0.8\r\n" + "cookie:textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
    assertMemoryLeak(() -> {
        HttpServerConfiguration httpServerConfiguration = new DefaultHttpServerConfiguration();
        SOCountDownLatch connectLatch = new SOCountDownLatch(1);
        SOCountDownLatch contextClosedLatch = new SOCountDownLatch(1);
        SOCountDownLatch requestReceivedLatch = new SOCountDownLatch(1);
        AtomicInteger closeCount = new AtomicInteger(0);
        try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration(), new IOContextFactory<HttpConnectionContext>() {

            @Override
            public HttpConnectionContext newInstance(long fd, IODispatcher<HttpConnectionContext> dispatcher1) {
                connectLatch.countDown();
                return new HttpConnectionContext(httpServerConfiguration.getHttpContextConfiguration()) {

                    @Override
                    public void close() {
                        // context is closed
                        if (closeCount.incrementAndGet() == 1) {
                            super.close();
                            contextClosedLatch.countDown();
                        }
                    }
                }.of(fd, dispatcher1);
            }
        })) {
            StringSink sink = new StringSink();
            final HttpRequestProcessorSelector selector = new HttpRequestProcessorSelector() {

                @Override
                public HttpRequestProcessor select(CharSequence url) {
                    return new HttpRequestProcessor() {

                        @Override
                        public void onHeadersReady(HttpConnectionContext context) {
                            HttpRequestHeader headers = context.getRequestHeader();
                            sink.put(headers.getMethodLine());
                            sink.put("\r\n");
                            ObjList<CharSequence> headerNames = headers.getHeaderNames();
                            for (int i = 0, n = headerNames.size(); i < n; i++) {
                                sink.put(headerNames.getQuick(i)).put(':');
                                sink.put(headers.getHeader(headerNames.getQuick(i)));
                                sink.put("\r\n");
                            }
                            sink.put("\r\n");
                            requestReceivedLatch.countDown();
                        }
                    };
                }

                @Override
                public HttpRequestProcessor getDefaultProcessor() {
                    return null;
                }

                @Override
                public void close() {
                }
            };
            AtomicBoolean serverRunning = new AtomicBoolean(true);
            SOCountDownLatch serverHaltLatch = new SOCountDownLatch(1);
            new Thread(() -> {
                while (serverRunning.get()) {
                    dispatcher.run(0);
                    dispatcher.processIOQueue((operation, context) -> context.handleClientOperation(operation, selector, EmptyRescheduleContext));
                }
                serverHaltLatch.countDown();
            }).start();
            long fd = Net.socketTcp(true);
            try {
                long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                try {
                    TestUtils.assertConnect(fd, sockAddr);
                    connectLatch.await();
                    int len = request.length();
                    long buffer = TestUtils.toMemory(request);
                    try {
                        Assert.assertEquals(len, Net.send(fd, buffer, len));
                    } finally {
                        Unsafe.free(buffer, len, MemoryTag.NATIVE_DEFAULT);
                    }
                    // do not disconnect right away, wait for server to receive the request
                    requestReceivedLatch.await();
                    Assert.assertEquals(0, Net.close(fd));
                    LOG.info().$("closed [fd=").$(fd).$(']').$();
                    fd = -1;
                    contextClosedLatch.await();
                    serverRunning.set(false);
                    serverHaltLatch.await();
                    Assert.assertEquals(0, dispatcher.getConnectionCount());
                    TestUtils.assertEquals(expected, sink);
                } finally {
                    Net.freeSockAddr(sockAddr);
                }
            } finally {
                if (fd != -1) {
                    Net.close(fd);
                }
            }
            Assert.assertEquals(1, closeCount.get());
        }
    });
}
Also used : IOOperation(io.questdb.network.IOOperation) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration) Log(io.questdb.log.Log) WorkerPoolConfiguration(io.questdb.mp.WorkerPoolConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestUtils.assertMemoryLeak(io.questdb.test.tools.TestUtils.assertMemoryLeak) WorkerPool(io.questdb.mp.WorkerPool) Timestamps(io.questdb.std.datetime.microtime.Timestamps) TestLatchedCounterFunctionFactory(io.questdb.griffin.engine.functions.test.TestLatchedCounterFunctionFactory) QueryCache(io.questdb.cutlass.http.processors.QueryCache) io.questdb.std(io.questdb.std) IODispatchers(io.questdb.network.IODispatchers) CyclicBarrier(java.util.concurrent.CyclicBarrier) SOCountDownLatch(io.questdb.mp.SOCountDownLatch) LockSupport(java.util.concurrent.locks.LockSupport) StaticContentProcessor(io.questdb.cutlass.http.processors.StaticContentProcessor) CountDownLatch(java.util.concurrent.CountDownLatch) Path(io.questdb.std.str.Path) io.questdb.cairo(io.questdb.cairo) AllowAllCairoSecurityContext(io.questdb.cairo.security.AllowAllCairoSecurityContext) IORequestProcessor(io.questdb.network.IORequestProcessor) NotNull(org.jetbrains.annotations.NotNull) JsonQueryProcessor(io.questdb.cutlass.http.processors.JsonQueryProcessor) TextImportProcessor(io.questdb.cutlass.http.processors.TextImportProcessor) IODispatcher(io.questdb.network.IODispatcher) AbstractCharSequence(io.questdb.std.str.AbstractCharSequence) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MPSequence(io.questdb.mp.MPSequence) IOContextFactory(io.questdb.network.IOContextFactory) NetUtils(io.questdb.cutlass.NetUtils) SqlException(io.questdb.griffin.SqlException) SCSequence(io.questdb.mp.SCSequence) IOContext(io.questdb.network.IOContext) TestUtils(io.questdb.test.tools.TestUtils) SqlCompiler(io.questdb.griffin.SqlCompiler) LogFactory(io.questdb.log.LogFactory) Metrics(io.questdb.Metrics) HealthCheckProcessor(io.questdb.cutlass.http.processors.HealthCheckProcessor) NetworkFacadeImpl(io.questdb.network.NetworkFacadeImpl) SqlExecutionContextImpl(io.questdb.griffin.SqlExecutionContextImpl) NetworkFacade(io.questdb.network.NetworkFacade) RingQueue(io.questdb.mp.RingQueue) PeerDisconnectedException(io.questdb.network.PeerDisconnectedException) TimeUnit(java.util.concurrent.TimeUnit) SharedRandom(io.questdb.griffin.engine.functions.rnd.SharedRandom) AtomicLong(java.util.concurrent.atomic.AtomicLong) PeerIsSlowToReadException(io.questdb.network.PeerIsSlowToReadException) StringSink(io.questdb.std.str.StringSink) Net(io.questdb.network.Net) MillisecondClock(io.questdb.std.datetime.millitime.MillisecondClock) IODispatcherConfiguration(io.questdb.network.IODispatcherConfiguration) org.junit(org.junit) SqlExecutionContext(io.questdb.griffin.SqlExecutionContext) TemporaryFolder(org.junit.rules.TemporaryFolder) ByteSequence(io.questdb.std.str.ByteSequence) InputStream(java.io.InputStream) AbstractCharSequence(io.questdb.std.str.AbstractCharSequence) StringSink(io.questdb.std.str.StringSink) SOCountDownLatch(io.questdb.mp.SOCountDownLatch) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration)

Example 7 with IODispatcher

use of io.questdb.network.IODispatcher in project questdb by bluestreak01.

the class IODispatcherTest method testConnectDisconnect.

@Test
public void testConnectDisconnect() throws Exception {
    LOG.info().$("started testConnectDisconnect").$();
    assertMemoryLeak(() -> {
        HttpServerConfiguration httpServerConfiguration = new DefaultHttpServerConfiguration();
        SOCountDownLatch connectLatch = new SOCountDownLatch(1);
        SOCountDownLatch contextClosedLatch = new SOCountDownLatch(1);
        AtomicInteger closeCount = new AtomicInteger(0);
        try (IODispatcher<HttpConnectionContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration(), new IOContextFactory<HttpConnectionContext>() {

            @Override
            public HttpConnectionContext newInstance(long fd, IODispatcher<HttpConnectionContext> dispatcher1) {
                connectLatch.countDown();
                return new HttpConnectionContext(httpServerConfiguration.getHttpContextConfiguration()) {

                    @Override
                    public void close() {
                        // context is closed
                        if (closeCount.incrementAndGet() == 1) {
                            super.close();
                            contextClosedLatch.countDown();
                        }
                    }
                }.of(fd, dispatcher1);
            }
        })) {
            HttpRequestProcessorSelector selector = new HttpRequestProcessorSelector() {

                @Override
                public HttpRequestProcessor select(CharSequence url) {
                    return null;
                }

                @Override
                public HttpRequestProcessor getDefaultProcessor() {
                    return new HttpRequestProcessor() {
                    };
                }

                @Override
                public void close() {
                }
            };
            AtomicBoolean serverRunning = new AtomicBoolean(true);
            SOCountDownLatch serverHaltLatch = new SOCountDownLatch(1);
            new Thread(() -> {
                while (serverRunning.get()) {
                    dispatcher.run(0);
                    dispatcher.processIOQueue((operation, context) -> context.handleClientOperation(operation, selector, EmptyRescheduleContext));
                }
                serverHaltLatch.countDown();
            }).start();
            long fd = Net.socketTcp(true);
            try {
                long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                try {
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        connectLatch.await();
                        Assert.assertEquals(0, Net.close(fd));
                        LOG.info().$("closed [fd=").$(fd).$(']').$();
                        fd = -1;
                        contextClosedLatch.await();
                    } finally {
                        serverRunning.set(false);
                        serverHaltLatch.await();
                    }
                    Assert.assertEquals(0, dispatcher.getConnectionCount());
                } finally {
                    Net.freeSockAddr(sockAddr);
                }
            } finally {
                if (fd != -1) {
                    Net.close(fd);
                }
            }
            Assert.assertEquals(1, closeCount.get());
        }
    });
}
Also used : IOOperation(io.questdb.network.IOOperation) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration) Log(io.questdb.log.Log) WorkerPoolConfiguration(io.questdb.mp.WorkerPoolConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestUtils.assertMemoryLeak(io.questdb.test.tools.TestUtils.assertMemoryLeak) WorkerPool(io.questdb.mp.WorkerPool) Timestamps(io.questdb.std.datetime.microtime.Timestamps) TestLatchedCounterFunctionFactory(io.questdb.griffin.engine.functions.test.TestLatchedCounterFunctionFactory) QueryCache(io.questdb.cutlass.http.processors.QueryCache) io.questdb.std(io.questdb.std) IODispatchers(io.questdb.network.IODispatchers) CyclicBarrier(java.util.concurrent.CyclicBarrier) SOCountDownLatch(io.questdb.mp.SOCountDownLatch) LockSupport(java.util.concurrent.locks.LockSupport) StaticContentProcessor(io.questdb.cutlass.http.processors.StaticContentProcessor) CountDownLatch(java.util.concurrent.CountDownLatch) Path(io.questdb.std.str.Path) io.questdb.cairo(io.questdb.cairo) AllowAllCairoSecurityContext(io.questdb.cairo.security.AllowAllCairoSecurityContext) IORequestProcessor(io.questdb.network.IORequestProcessor) NotNull(org.jetbrains.annotations.NotNull) JsonQueryProcessor(io.questdb.cutlass.http.processors.JsonQueryProcessor) TextImportProcessor(io.questdb.cutlass.http.processors.TextImportProcessor) IODispatcher(io.questdb.network.IODispatcher) AbstractCharSequence(io.questdb.std.str.AbstractCharSequence) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MPSequence(io.questdb.mp.MPSequence) IOContextFactory(io.questdb.network.IOContextFactory) NetUtils(io.questdb.cutlass.NetUtils) SqlException(io.questdb.griffin.SqlException) SCSequence(io.questdb.mp.SCSequence) IOContext(io.questdb.network.IOContext) TestUtils(io.questdb.test.tools.TestUtils) SqlCompiler(io.questdb.griffin.SqlCompiler) LogFactory(io.questdb.log.LogFactory) Metrics(io.questdb.Metrics) HealthCheckProcessor(io.questdb.cutlass.http.processors.HealthCheckProcessor) NetworkFacadeImpl(io.questdb.network.NetworkFacadeImpl) SqlExecutionContextImpl(io.questdb.griffin.SqlExecutionContextImpl) NetworkFacade(io.questdb.network.NetworkFacade) RingQueue(io.questdb.mp.RingQueue) PeerDisconnectedException(io.questdb.network.PeerDisconnectedException) TimeUnit(java.util.concurrent.TimeUnit) SharedRandom(io.questdb.griffin.engine.functions.rnd.SharedRandom) AtomicLong(java.util.concurrent.atomic.AtomicLong) PeerIsSlowToReadException(io.questdb.network.PeerIsSlowToReadException) StringSink(io.questdb.std.str.StringSink) Net(io.questdb.network.Net) MillisecondClock(io.questdb.std.datetime.millitime.MillisecondClock) IODispatcherConfiguration(io.questdb.network.IODispatcherConfiguration) org.junit(org.junit) SqlExecutionContext(io.questdb.griffin.SqlExecutionContext) TemporaryFolder(org.junit.rules.TemporaryFolder) ByteSequence(io.questdb.std.str.ByteSequence) InputStream(java.io.InputStream) AbstractCharSequence(io.questdb.std.str.AbstractCharSequence) SOCountDownLatch(io.questdb.mp.SOCountDownLatch) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration)

Example 8 with IODispatcher

use of io.questdb.network.IODispatcher in project questdb by bluestreak01.

the class IODispatcherTest method testBiasWrite.

@Test
public void testBiasWrite() throws Exception {
    LOG.info().$("started testBiasWrite").$();
    assertMemoryLeak(() -> {
        SOCountDownLatch connectLatch = new SOCountDownLatch(1);
        SOCountDownLatch contextClosedLatch = new SOCountDownLatch(1);
        try (IODispatcher<HelloContext> dispatcher = IODispatchers.create(new DefaultIODispatcherConfiguration() {

            @Override
            public int getInitialBias() {
                return IODispatcherConfiguration.BIAS_WRITE;
            }

            @Override
            public boolean getPeerNoLinger() {
                return false;
            }
        }, (fd, dispatcher1) -> {
            connectLatch.countDown();
            return new HelloContext(fd, contextClosedLatch, dispatcher1);
        })) {
            AtomicBoolean serverRunning = new AtomicBoolean(true);
            SOCountDownLatch serverHaltLatch = new SOCountDownLatch(1);
            new Thread(() -> {
                while (serverRunning.get()) {
                    dispatcher.run(0);
                    dispatcher.processIOQueue((operation, context) -> {
                        if (operation == IOOperation.WRITE) {
                            Assert.assertEquals(1024, Net.send(context.getFd(), context.buffer, 1024));
                            context.dispatcher.disconnect(context, IODispatcher.DISCONNECT_REASON_TEST);
                        }
                    });
                }
                serverHaltLatch.countDown();
            }).start();
            long fd = Net.socketTcp(true);
            try {
                long sockAddr = Net.sockaddr("127.0.0.1", 9001);
                try {
                    try {
                        TestUtils.assertConnect(fd, sockAddr);
                        connectLatch.await();
                        long buffer = Unsafe.malloc(2048, MemoryTag.NATIVE_DEFAULT);
                        try {
                            Assert.assertEquals(1024, Net.recv(fd, buffer, 1024));
                        } finally {
                            Unsafe.free(buffer, 2048, MemoryTag.NATIVE_DEFAULT);
                        }
                        Assert.assertEquals(0, Net.close(fd));
                        LOG.info().$("closed [fd=").$(fd).$(']').$();
                        fd = -1;
                        contextClosedLatch.await();
                    } finally {
                        serverRunning.set(false);
                        serverHaltLatch.await();
                    }
                    Assert.assertEquals(0, dispatcher.getConnectionCount());
                } finally {
                    Net.freeSockAddr(sockAddr);
                }
            } finally {
                if (fd != -1) {
                    Net.close(fd);
                }
            }
        }
    });
}
Also used : IOOperation(io.questdb.network.IOOperation) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration) Log(io.questdb.log.Log) WorkerPoolConfiguration(io.questdb.mp.WorkerPoolConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestUtils.assertMemoryLeak(io.questdb.test.tools.TestUtils.assertMemoryLeak) WorkerPool(io.questdb.mp.WorkerPool) Timestamps(io.questdb.std.datetime.microtime.Timestamps) TestLatchedCounterFunctionFactory(io.questdb.griffin.engine.functions.test.TestLatchedCounterFunctionFactory) QueryCache(io.questdb.cutlass.http.processors.QueryCache) io.questdb.std(io.questdb.std) IODispatchers(io.questdb.network.IODispatchers) CyclicBarrier(java.util.concurrent.CyclicBarrier) SOCountDownLatch(io.questdb.mp.SOCountDownLatch) LockSupport(java.util.concurrent.locks.LockSupport) StaticContentProcessor(io.questdb.cutlass.http.processors.StaticContentProcessor) CountDownLatch(java.util.concurrent.CountDownLatch) Path(io.questdb.std.str.Path) io.questdb.cairo(io.questdb.cairo) AllowAllCairoSecurityContext(io.questdb.cairo.security.AllowAllCairoSecurityContext) IORequestProcessor(io.questdb.network.IORequestProcessor) NotNull(org.jetbrains.annotations.NotNull) JsonQueryProcessor(io.questdb.cutlass.http.processors.JsonQueryProcessor) TextImportProcessor(io.questdb.cutlass.http.processors.TextImportProcessor) IODispatcher(io.questdb.network.IODispatcher) AbstractCharSequence(io.questdb.std.str.AbstractCharSequence) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MPSequence(io.questdb.mp.MPSequence) IOContextFactory(io.questdb.network.IOContextFactory) NetUtils(io.questdb.cutlass.NetUtils) SqlException(io.questdb.griffin.SqlException) SCSequence(io.questdb.mp.SCSequence) IOContext(io.questdb.network.IOContext) TestUtils(io.questdb.test.tools.TestUtils) SqlCompiler(io.questdb.griffin.SqlCompiler) LogFactory(io.questdb.log.LogFactory) Metrics(io.questdb.Metrics) HealthCheckProcessor(io.questdb.cutlass.http.processors.HealthCheckProcessor) NetworkFacadeImpl(io.questdb.network.NetworkFacadeImpl) SqlExecutionContextImpl(io.questdb.griffin.SqlExecutionContextImpl) NetworkFacade(io.questdb.network.NetworkFacade) RingQueue(io.questdb.mp.RingQueue) PeerDisconnectedException(io.questdb.network.PeerDisconnectedException) TimeUnit(java.util.concurrent.TimeUnit) SharedRandom(io.questdb.griffin.engine.functions.rnd.SharedRandom) AtomicLong(java.util.concurrent.atomic.AtomicLong) PeerIsSlowToReadException(io.questdb.network.PeerIsSlowToReadException) StringSink(io.questdb.std.str.StringSink) Net(io.questdb.network.Net) MillisecondClock(io.questdb.std.datetime.millitime.MillisecondClock) IODispatcherConfiguration(io.questdb.network.IODispatcherConfiguration) org.junit(org.junit) SqlExecutionContext(io.questdb.griffin.SqlExecutionContext) TemporaryFolder(org.junit.rules.TemporaryFolder) ByteSequence(io.questdb.std.str.ByteSequence) InputStream(java.io.InputStream) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) DefaultIODispatcherConfiguration(io.questdb.network.DefaultIODispatcherConfiguration) SOCountDownLatch(io.questdb.mp.SOCountDownLatch)

Aggregations

Metrics (io.questdb.Metrics)8 io.questdb.cairo (io.questdb.cairo)8 AllowAllCairoSecurityContext (io.questdb.cairo.security.AllowAllCairoSecurityContext)8 NetUtils (io.questdb.cutlass.NetUtils)8 HealthCheckProcessor (io.questdb.cutlass.http.processors.HealthCheckProcessor)8 JsonQueryProcessor (io.questdb.cutlass.http.processors.JsonQueryProcessor)8 QueryCache (io.questdb.cutlass.http.processors.QueryCache)8 StaticContentProcessor (io.questdb.cutlass.http.processors.StaticContentProcessor)8 TextImportProcessor (io.questdb.cutlass.http.processors.TextImportProcessor)8 SqlCompiler (io.questdb.griffin.SqlCompiler)8 SqlException (io.questdb.griffin.SqlException)8 SqlExecutionContext (io.questdb.griffin.SqlExecutionContext)8 SqlExecutionContextImpl (io.questdb.griffin.SqlExecutionContextImpl)8 SharedRandom (io.questdb.griffin.engine.functions.rnd.SharedRandom)8 TestLatchedCounterFunctionFactory (io.questdb.griffin.engine.functions.test.TestLatchedCounterFunctionFactory)8 Log (io.questdb.log.Log)8 LogFactory (io.questdb.log.LogFactory)8 MPSequence (io.questdb.mp.MPSequence)8 RingQueue (io.questdb.mp.RingQueue)8 SCSequence (io.questdb.mp.SCSequence)8