Search in sources :

Example 1 with HttpClientFactory

use of io.mantisrx.runtime.source.http.HttpClientFactory in project mantis by Netflix.

the class HttpSourceImplTest method testTimeoutShouldUnsubscribeServer.

@Test
public void testTimeoutShouldUnsubscribeServer() throws Exception {
    HttpClientFactory<ByteBuf, ByteBuf> factory = new HttpClientFactory<ByteBuf, ByteBuf>() {

        @Override
        public HttpClient<ByteBuf, ByteBuf> createClient(ServerInfo server) {
            ClientConfig clientConfig = new ClientConfig.Builder().readTimeout(10, TimeUnit.MILLISECONDS).build();
            return new HttpClientBuilder<ByteBuf, ByteBuf>(server.getHost(), server.getPort()).config(clientConfig).build();
        }
    };
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = HttpSourceImpl.builder(factory, HttpRequestFactories.createGetFactory("test/timeout?timeout=10000"), HttpSourceImpl.<ByteBuf>contextWrapper()).withActivityObserver(sourceObserver).resumeWith(new ClientResumePolicy<ByteBuf, ByteBuf>() {

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
            // TODO Auto-generated method stub
            return null;
        }

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
            // TODO Auto-generated method stub
            return null;
        }
    }).withServerProvider(new HttpServerProvider() {

        @Override
        public Observable<ServerInfo> getServersToAdd() {
            return Observable.from(localServerProvider.getServers()).map(new Func1<Server, ServerInfo>() {

                @Override
                public ServerInfo call(Server server) {
                    return toServerInfo(server);
                }
            });
        }

        @Override
        public Observable<ServerInfo> getServersToRemove() {
            return Observable.empty();
        }
    }).build();
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicReference<Throwable> ex = new AtomicReference<>();
    final AtomicReference<ServerContext<ByteBuf>> items = new AtomicReference<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).subscribe(new Subscriber<ServerContext<ByteBuf>>() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public void onError(Throwable e) {
            ex.set(e);
            latch.countDown();
        }

        @Override
        public void onNext(ServerContext<ByteBuf> pair) {
            items.set(pair);
        }
    });
    if (!latch.await(5, TimeUnit.SECONDS)) {
        fail("The test case should finish way sooner than 5 seconds. ");
    }
    Assert.assertNull("The timeout error should be captured by the client so it does not surface to the source", ex.get());
    Assert.assertNull("There should be no emitted item due to connection timeout", items.get());
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        assertEquals("There should be no source level error", 0, sourceObserver.getErrorCount());
        assertEquals("There should be one connection attempt per server", 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ATTEMPTED));
        assertEquals("There should be no established connection per server due to read timeout. ", 0, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ESTABLISHED));
        assertEquals("There should no subscribed server because of read timeout", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
    }
}
Also used : Server(io.mantisrx.runtime.source.http.LocalServerProvider.Server) ServerInfo(mantis.io.reactivex.netty.client.RxClient.ServerInfo) HttpClientBuilder(mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder) Builder(io.mantisrx.runtime.source.http.impl.HttpSourceImpl.Builder) Index(io.mantisrx.runtime.source.Index) HttpClientBuilder(mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder) ByteBuf(io.netty.buffer.ByteBuf) ClientConfig(mantis.io.reactivex.netty.client.RxClient.ClientConfig) HttpClientFactory(io.mantisrx.runtime.source.http.HttpClientFactory) Context(io.mantisrx.runtime.Context) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) Observable(rx.Observable) HttpServerProvider(io.mantisrx.runtime.source.http.HttpServerProvider) Test(org.junit.Test)

Example 2 with HttpClientFactory

use of io.mantisrx.runtime.source.http.HttpClientFactory in project mantis by Netflix.

the class HttpSourceImplTest method testConnectionFailureShouldBeCaptured.

@Test
public void testConnectionFailureShouldBeCaptured() throws Exception {
    HttpClientFactory<ByteBuf, ByteBuf> factory = new HttpClientFactory<ByteBuf, ByteBuf>() {

        @Override
        public HttpClient<ByteBuf, ByteBuf> createClient(ServerInfo server) {
            HttpClientBuilder<ByteBuf, ByteBuf> clientBuilder = new HttpClientBuilder<>(server.getHost(), server.getPort());
            return clientBuilder.channelOption(ChannelOption.CONNECT_TIMEOUT_MILLIS, 100).build();
        }
    };
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = HttpSourceImpl.builder(factory, HttpRequestFactories.createGetFactory("/"), HttpSourceImpl.<ByteBuf>contextWrapper()).withActivityObserver(sourceObserver).withServerProvider(new HttpServerProvider() {

        @Override
        public Observable<ServerInfo> getServersToAdd() {
            return Observable.just(new ServerInfo("localhost", UNUSED_PORT));
        }

        @Override
        public Observable<ServerInfo> getServersToRemove() {
            return Observable.empty();
        }
    }).build();
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicReference<Throwable> ex = new AtomicReference<>();
    final AtomicReference<ServerContext<ByteBuf>> items = new AtomicReference<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).subscribe(new Subscriber<ServerContext<ByteBuf>>() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public void onError(Throwable e) {
            ex.set(e);
            latch.countDown();
        }

        @Override
        public void onNext(ServerContext<ByteBuf> pair) {
            items.set(pair);
        }
    });
    if (!latch.await(5, TimeUnit.HOURS)) {
        fail("The test case should finish way sooner than 5 seconds. ");
    }
    Assert.assertNull("The connection error should be captured per server, therefore not propagated up to the entire source.", ex.get());
    Assert.assertNull("There should be no emitted item due to connection timeout", items.get());
}
Also used : Context(io.mantisrx.runtime.Context) ServerInfo(mantis.io.reactivex.netty.client.RxClient.ServerInfo) AtomicReference(java.util.concurrent.atomic.AtomicReference) Index(io.mantisrx.runtime.source.Index) HttpClientBuilder(mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder) ByteBuf(io.netty.buffer.ByteBuf) CountDownLatch(java.util.concurrent.CountDownLatch) HttpServerProvider(io.mantisrx.runtime.source.http.HttpServerProvider) HttpClientFactory(io.mantisrx.runtime.source.http.HttpClientFactory) Test(org.junit.Test)

Example 3 with HttpClientFactory

use of io.mantisrx.runtime.source.http.HttpClientFactory in project mantis by Netflix.

the class HttpSourceImplTest method testResumeOnTimeout.

@Test
public void testResumeOnTimeout() throws Exception {
    HttpClientFactory<ByteBuf, ByteBuf> factory = new HttpClientFactory<ByteBuf, ByteBuf>() {

        @Override
        public HttpClient<ByteBuf, ByteBuf> createClient(ServerInfo server) {
            ClientConfig clientConfig = new ClientConfig.Builder().readTimeout(10, TimeUnit.MILLISECONDS).build();
            return new HttpClientBuilder<ByteBuf, ByteBuf>(server.getHost(), server.getPort()).config(clientConfig).build();
        }
    };
    final ConcurrentMap<ServerInfo, AtomicInteger> resumptions = new ConcurrentHashMap<>();
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        resumptions.put(server, new AtomicInteger());
    }
    final int maxRepeat = 5;
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = HttpSourceImpl.builder(factory, HttpRequestFactories.createGetFactory("test/timeout?timeout=10000"), HttpSourceImpl.<ByteBuf>contextWrapper()).withActivityObserver(sourceObserver).resumeWith(new ClientResumePolicy<ByteBuf, ByteBuf>() {

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
            if (attempts <= maxRepeat) {
                resumptions.get(clientContext.getServer()).incrementAndGet();
                return clientContext.newResponse();
            }
            return null;
        }

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
            return null;
        }
    }).withServerProvider(localServerProvider).build();
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicReference<Throwable> ex = new AtomicReference<>();
    final AtomicReference<ServerContext<ByteBuf>> items = new AtomicReference<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).subscribe(new Subscriber<ServerContext<ByteBuf>>() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public void onError(Throwable e) {
            ex.set(e);
            latch.countDown();
        }

        @Override
        public void onNext(ServerContext<ByteBuf> pair) {
            items.set(pair);
        }
    });
    if (!latch.await(10, TimeUnit.SECONDS)) {
        fail("The test case should finish way sooner than 10 seconds. ");
    }
    Assert.assertNull("The timeout error should be captured by the client so it does not surface to the source", ex.get());
    Assert.assertNull("There should be no emitted item due to connection timeout", items.get());
    for (ServerInfo serverInfo : localServerProvider.getServerInfos()) {
        assertEquals("There should be no source level error", 0, sourceObserver.getErrorCount());
        assertEquals("There should be one connection attempt per server per retry", maxRepeat + 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ATTEMPTED));
        assertEquals("There should be no established connection per server due to read timeout. ", 0, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ESTABLISHED));
        assertEquals("There should no subscribed server because of read timeout", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
        assertEquals("Each server will repeat exactly " + maxRepeat + " times", maxRepeat, resumptions.get(serverInfo).get());
    }
}
Also used : ServerInfo(mantis.io.reactivex.netty.client.RxClient.ServerInfo) HttpClientBuilder(mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder) Builder(io.mantisrx.runtime.source.http.impl.HttpSourceImpl.Builder) Index(io.mantisrx.runtime.source.Index) HttpClientBuilder(mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder) ByteBuf(io.netty.buffer.ByteBuf) ClientConfig(mantis.io.reactivex.netty.client.RxClient.ClientConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HttpClientFactory(io.mantisrx.runtime.source.http.HttpClientFactory) Context(io.mantisrx.runtime.Context) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) Observable(rx.Observable) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.junit.Test)

Aggregations

Context (io.mantisrx.runtime.Context)3 Index (io.mantisrx.runtime.source.Index)3 HttpClientFactory (io.mantisrx.runtime.source.http.HttpClientFactory)3 ByteBuf (io.netty.buffer.ByteBuf)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 AtomicReference (java.util.concurrent.atomic.AtomicReference)3 ServerInfo (mantis.io.reactivex.netty.client.RxClient.ServerInfo)3 HttpClientBuilder (mantis.io.reactivex.netty.protocol.http.client.HttpClientBuilder)3 Test (org.junit.Test)3 HttpServerProvider (io.mantisrx.runtime.source.http.HttpServerProvider)2 Builder (io.mantisrx.runtime.source.http.impl.HttpSourceImpl.Builder)2 ClientConfig (mantis.io.reactivex.netty.client.RxClient.ClientConfig)2 Observable (rx.Observable)2 Server (io.mantisrx.runtime.source.http.LocalServerProvider.Server)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1