Examples with EventType io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType used on opensource projects

Example 1 with EventType

use of io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType in project mantis by Netflix.

the class HttpSourceImplTest method testResumeOnCompletionButNotOnRemovedServers.

@Test
public void testResumeOnCompletionButNotOnRemovedServers() throws Exception {
    final int maxRepeat = 10;
    final int cutOff = 5;
    final CountDownLatch countGate = new CountDownLatch(localServerProvider.serverSize() * (cutOff - 1));
    final ConcurrentHashMap<ServerInfo, AtomicInteger> resumptionCounts = new ConcurrentHashMap<>();
    final ConcurrentHashMap<ServerInfo, AtomicInteger> counterPerServer = new ConcurrentHashMap<>();
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        resumptionCounts.put(serverInfo, new AtomicInteger(0));
        counterPerServer.put(serverInfo, new AtomicInteger(0));
    }
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = createSingleEntitySource(new ClientResumePolicy<ByteBuf, ByteBuf>() {

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
            return null;
        }

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
            resumptionCounts.get(clientContext.getServer()).incrementAndGet();
            countGate.countDown();
            if (attempts < maxRepeat) {
                return clientContext.newResponse();
            } else {
                return null;
            }
        }
    });
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    Observable.merge(source.call(new Context(), new Index(1, 1))).map(new Func1<ServerContext<ByteBuf>, ServerContext<String>>() {

        @Override
        public ServerContext<String> call(ServerContext<ByteBuf> pair) {
            return new ServerContext<>(pair.getServer(), pair.getValue().toString(Charset.defaultCharset()));
        }
    }).doOnNext(new Action1<ServerContext<String>>() {

        @Override
        public void call(ServerContext<String> context) {
            counter.incrementAndGet();
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, context.getValue());
            ServerInfo server = context.getServer();
            counterPerServer.get(server).incrementAndGet();
            if (counterPerServer.get(server).get() > cutOff) {
                localServerProvider.removeServer(server);
            }
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            try {
                countGate.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            done.countDown();
        }
    }).subscribe();
    long wait = 5;
    if (!done.await(wait, TimeUnit.SECONDS)) {
        fail(String.format("All streaming should be done within %d seconds. ", wait));
    }
    assertEquals("All server should be resumed", localServerProvider.serverSize(), resumptionCounts.size());
    for (ServerInfo server : resumptionCounts.keySet()) {
        assertTrue(String.format("The server %s:%s should be resumed fewer than %d times", server.getHost(), server.getPort(), maxRepeat), maxRepeat > resumptionCounts.get(server).get());
    }
    assertTrue(String.format("There should be at least %d response from each of the %d servers", cutOff + 1, localServerProvider.serverSize()), (cutOff + 1) * localServerProvider.serverSize() <= counter.get());
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        assertEquals("There should be no error", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_FAILED));
    }
    assertEquals(String.format("There are %d repeats, but there should be only one final completion", maxRepeat - 1), 1, sourceObserver.getCompletionCount());
    assertEquals("There should be no error", 0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    Set<EventType> expectedEvents = new HashSet<>();
    expectedEvents.addAll(Arrays.asList(EventType.SUBSCRIPTION_CANCELED, EventType.SERVER_FOUND, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ENDED, EventType.CONNECTION_UNSUBSCRIBED));
    assertEquals(expectedEvents, events);
    assertEquals("Each connection should be unsubscribed once by the subscriber", localServerProvider.serverSize(), sourceObserver.getEventCount(CONNECTION_UNSUBSCRIBED));
    for (EventType eventType : new EventType[] { EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED }) {
        assertTrue(String.format("Event %s should be recorded at least %d times per server", eventType, cutOff), (cutOff + 1) * localServerProvider.serverSize() <= sourceObserver.getEventCount(eventType));
    }
    for (EventType eventType : new EventType[] { EventType.SERVER_FOUND, EventType.SUBSCRIPTION_CANCELED }) {
        assertEquals(String.format("Event %s should be recorded exactly once per server", eventType), localServerProvider.serverSize(), sourceObserver.getEventCount(eventType));
    }
}

Example 2 with EventType

use of io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType in project mantis by Netflix.

the class HttpSourceImplTest method testResumeOnCompletion.

@Test
public void testResumeOnCompletion() throws Exception {
    final int maxRepeat = 10;
    final CountDownLatch countGate = new CountDownLatch(maxRepeat * localServerProvider.serverSize());
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = createSingleEntitySource(new ClientResumePolicy<ByteBuf, ByteBuf>() {

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
            return null;
        }

        @Override
        public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
            countGate.countDown();
            if (attempts < maxRepeat) {
                return clientContext.newResponse();
            } else {
                return null;
            }
        }
    });
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    Observable.merge(source.call(new Context(), new Index(1, 1))).map(new Func1<ServerContext<ByteBuf>, ServerContext<String>>() {

        @Override
        public ServerContext<String> call(ServerContext<ByteBuf> pair) {
            try {
                return new ServerContext<>(pair.getServer(), pair.getValue().retain().toString(Charset.defaultCharset()));
            } catch (Throwable e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        }
    }).doOnNext(new Action1<ServerContext<String>>() {

        @Override
        public void call(ServerContext<String> pair) {
            counter.incrementAndGet();
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, pair.getValue());
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            try {
                countGate.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            done.countDown();
        }
    }).subscribe();
    long wait = 5;
    if (!done.await(5, TimeUnit.SECONDS)) {
        fail(String.format("All streaming should be done within %d seconds. ", wait));
    }
    assertEquals(String.format("There should be exactly %d response from each of the %d servers", maxRepeat, localServerProvider.serverSize()), maxRepeat * localServerProvider.serverSize(), counter.get());
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        assertEquals(String.format("There should be %d completion per server as resumption function should called %d times", maxRepeat, maxRepeat - 1), maxRepeat, sourceObserver.getCount(serverInfo, EventType.SOURCE_COMPLETED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_FAILED));
        assertEquals(String.format("Connection per server should have been established %d times", maxRepeat), 10, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
    }
    assertEquals(String.format("There are %d repeats, but there should be only one final completion", maxRepeat - 1), 1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    assertEquals("Each connection should be unsubscribed once by the subscriber", localServerProvider.serverSize(), sourceObserver.getEventCount(CONNECTION_UNSUBSCRIBED));
    for (EventType eventType : new EventType[] { EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED }) {
        assertEquals(String.format("Event %s should be recorded exactly %d times per server", eventType, maxRepeat), maxRepeat * localServerProvider.serverSize(), sourceObserver.getEventCount(eventType));
    }
    assertEquals("Event SERVER_FOUND should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(EventType.SERVER_FOUND));
}

Example 3 with EventType

use of io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType in project mantis by Netflix.

the class HttpSourceTest method sourceEchoStreamFromPost.

private void sourceEchoStreamFromPost(HttpSource<ServerSentEvent, ServerSentEvent> source, String postContent) throws Exception {
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            counter.incrementAndGet();
            String msg = event.contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 30000;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single echo", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), counter.get());
    assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(postContent).get());
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}

Example 4 with EventType

use of io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType in project mantis by Netflix.

the class ContextualHttpSourceTest method canStreamFromMultipleServersWithCorrectContext.

@Test
public void canStreamFromMultipleServersWithCorrectContext() throws Exception {
    ContextualHttpSource<ServerSentEvent> source = HttpSources.contextualSource(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    final CopyOnWriteArraySet<ServerInfo> connectedServers = new CopyOnWriteArraySet<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {

        @Override
        public void call(ServerContext<ServerSentEvent> pair) {
            assertTrue(pair.getValue().contentAsString().contains("line"));
            counter.incrementAndGet();
            String msg = pair.getValue().contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
            connectedServers.add(pair.getServer());
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            System.out.println("completed");
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    assertEquals("There should be as many as provided servers", localServerProvider.serverSize(), connectedServers.size());
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals("There should be one completions", 1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}

Example 5 with EventType

use of io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType in project mantis by Netflix.

the class HttpSourceTest method pollingSourceWillWork.

@Test
public void pollingSourceWillWork() throws Exception {
    ServerInfo server = localServerProvider.getServerInfos().get(0);
    HttpSource<ByteBuf, String> source = HttpSources.pollingSource(server.getHost(), server.getPort(), "test/singleEntity").withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final int maxRepeat = 10;
    final CountDownLatch done = new CountDownLatch(maxRepeat);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Subscription subscription = Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<String>() {

        @Override
        public void call(String content) {
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, content);
            counter.incrementAndGet();
            result.putIfAbsent(content, new AtomicInteger());
            result.get(content).incrementAndGet();
            done.countDown();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), maxRepeat);
    assertTrue(String.format("There should be at least %d completions after %d repeats (The last one may not have completion. Actual completion count: %d", maxRepeat - 1, maxRepeat, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED)), maxRepeat - 1 <= sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
    assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
    assertEquals("There should be " + maxRepeat + " connection establishment in total", maxRepeat, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    assertEquals("There should no final completion", 0, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertTrue("Polling Source always has subscriptions, so there won't be subscription_ended event. But other events should all be there", EXPECTED_EVENTS_SETS.containsAll(events));
}