Examples with ServerSentEvent mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent used on opensource projects

Example 6 with ServerSentEvent

use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.

the class HttpSourceTest method sourcingStream.

private void sourcingStream(HttpSource<ServerSentEvent, ServerSentEvent> source) throws InterruptedException {
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            counter.incrementAndGet();
            String msg = event.contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 30000;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}

Example 7 with ServerSentEvent

use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.

the class HttpSourceTest method testResubscribeShouldAlwaysWork.

@Test
public void testResubscribeShouldAlwaysWork() throws Exception {
    HttpSource<ServerSentEvent, ServerSentEvent> source = HttpSources.source(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    int totalCount = 5;
    final CountDownLatch latch = new CountDownLatch(totalCount);
    Observable<ServerSentEvent> stream = Observable.merge(source.call(new Context(), new Index(1, 1)));
    Subscription sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            latch.countDown();
        }
    });
    long waitSeconds = 10;
    boolean countedDown = latch.await(waitSeconds, TimeUnit.SECONDS);
    if (!countedDown) {
        fail(String.format("Waited too long to receive %d events within %d seconds. Total counted: %d", totalCount, waitSeconds, latch.getCount()));
    }
    sub.unsubscribe();
    final CountDownLatch newLatch = new CountDownLatch(totalCount);
    sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            newLatch.countDown();
        }
    });
    countedDown = newLatch.await(5, TimeUnit.SECONDS);
    if (!countedDown) {
        fail("Waited too long to receive enough events. Counted: " + latch.getCount());
    }
    sub.unsubscribe();
}

Example 8 with ServerSentEvent

use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.

the class ServerSentEventRequestHandler method handle.

@Override
public Observable<Void> handle(HttpServerRequest<ByteBuf> request, final HttpServerResponse<ServerSentEvent> response) {
    InetSocketAddress socketAddress = (InetSocketAddress) response.getChannel().remoteAddress();
    LOG.info("HTTP SSE connection received from " + socketAddress.getAddress() + ":" + socketAddress.getPort() + "  queryParams: " + request.getQueryParameters());
    final String socketAddrStr = socketAddress.getAddress().toString();
    final WritableEndpoint<String> sn = new WritableEndpoint<>(socketAddress.getHostString(), socketAddress.getPort(), Endpoint.uniqueHost(socketAddress.getHostString(), socketAddress.getPort(), null));
    final Map<String, List<String>> queryParameters = request.getQueryParameters();
    final SlotAssignmentManager<String> slotMgr = ssm.registerServer(sn, queryParameters);
    final AtomicLong lastResponseFlush = new AtomicLong();
    lastResponseFlush.set(-1);
    final AtomicLong lastResponseSent = new AtomicLong(-1);
    // copy reference, then apply request specific filters, sampling
    Observable<T> requestObservable = observableToServe;
    // decouple the observable on a separate thread and add backpressure handling
    // ServiceRegistry.INSTANCE.getPropertiesService().getStringValue("sse.decouple", "false");
    String decoupleSSE = "false";
    if ("true".equals(decoupleSSE)) {
        final BasicTag sockAddrTag = new BasicTag("sockAddr", Optional.ofNullable(socketAddrStr).orElse("none"));
        requestObservable = requestObservable.lift(new DropOperator<T>("outgoing_ServerSentEventRequestHandler", sockAddrTag)).observeOn(Schedulers.io());
    }
    response.getHeaders().set("Access-Control-Allow-Origin", "*");
    response.getHeaders().set("content-type", "text/event-stream");
    response.getHeaders().set("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate");
    response.getHeaders().set("Pragma", "no-cache");
    response.flush();
    String uniqueClientId = socketAddrStr;
    if (queryParameters != null && queryParameters.containsKey(CLIENT_ID_PARAM)) {
        // enablePings
        uniqueClientId = queryParameters.get(CLIENT_ID_PARAM).get(0);
    }
    if (queryParameters != null && queryParameters.containsKey(FORMAT_PARAM)) {
        format = queryParameters.get(FORMAT_PARAM).get(0);
    }
    if (queryParameters != null && requestPreprocessor != null) {
        requestPreprocessor.call(queryParameters, context);
    }
    // apply sampling, milli, then seconds
    if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM_MSEC)) {
        // apply sampling rate
        int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM_MSEC).get(0));
        requestObservable = requestObservable.sample(samplingRate, TimeUnit.MILLISECONDS);
    }
    if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM)) {
        // apply sampling rate
        int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM).get(0));
        requestObservable = requestObservable.sample(samplingRate, TimeUnit.SECONDS);
    }
    if (queryParameters != null && queryParameters.containsKey(ENABLE_PINGS_PARAM)) {
        // enablePings
        String enablePings = queryParameters.get(ENABLE_PINGS_PARAM).get(0);
        if ("true".equalsIgnoreCase(enablePings)) {
            pingsEnabled = true;
        } else {
            pingsEnabled = false;
        }
    }
    if (queryParameters != null && queryParameters.containsKey("delay")) {
        // apply flush
        try {
            int flushInterval = Integer.parseInt(queryParameters.get("delay").get(0));
            if (flushInterval >= 50) {
                flushIntervalMillis = flushInterval;
            } else {
                LOG.warn("delay parameter too small " + flushInterval + " min. is 100");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    final byte[] delimiter = queryParameters != null && queryParameters.containsKey(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER) && queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0) != null ? queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0).getBytes() : null;
    Tag[] tags = new Tag[2];
    final String clientId = Optional.ofNullable(uniqueClientId).orElse("none");
    final String sockAddr = Optional.ofNullable(socketAddrStr).orElse("none");
    tags[0] = new BasicTag("clientId", clientId);
    tags[1] = new BasicTag("sockAddr", sockAddr);
    Metrics sseSinkMetrics = new Metrics.Builder().id("ServerSentEventRequestHandler", tags).addCounter("processedCounter").addCounter("pingCounter").addCounter("errorCounter").addCounter("droppedCounter").addCounter("flushCounter").build();
    final Counter msgProcessedCounter = sseSinkMetrics.getCounter("processedCounter");
    final Counter pingCounter = sseSinkMetrics.getCounter("pingCounter");
    final Counter errorCounter = sseSinkMetrics.getCounter("errorCounter");
    final Counter droppedWrites = sseSinkMetrics.getCounter("droppedCounter");
    final Counter flushCounter = sseSinkMetrics.getCounter("flushCounter");
    // get predicate, defaults to return true for all T
    Func1<T, Boolean> filterFunction = new Func1<T, Boolean>() {

        @Override
        public Boolean call(T t1) {
            return true;
        }
    };
    if (queryParameters != null && predicate != null) {
        filterFunction = predicate.getPredicate().call(queryParameters);
    }
    final Subscription timerSubscription = Observable.interval(1, TimeUnit.SECONDS).doOnNext(new Action1<Long>() {

        @Override
        public void call(Long t1) {
            long currentTime = System.currentTimeMillis();
            if (pingsEnabled && (lastResponseSent.get() == -1 || currentTime > lastResponseSent.get() + PING_INTERVAL)) {
                pingCounter.increment();
                response.writeStringAndFlush(PING);
                lastResponseSent.set(currentTime);
            }
        }
    }).subscribe();
    return requestObservable.filter(filterFunction).map(encoder).lift(new DisableBackPressureOperator<String>()).buffer(flushIntervalMillis, TimeUnit.MILLISECONDS).flatMap(new Func1<List<String>, Observable<Void>>() {

        @Override
        public Observable<Void> call(List<String> valueList) {
            if (response.isCloseIssued() || !response.getChannel().isActive()) {
                LOG.info("Client closed detected, throwing closed channel exception");
                return Observable.error(new ClosedChannelException());
            }
            List<String> filteredList = valueList.stream().filter(e -> {
                return slotMgr.filter(sn, e.getBytes());
            }).collect(Collectors.toList());
            if (response.getChannel().isWritable()) {
                flushCounter.increment();
                if (format.equals(BINARY_FORMAT)) {
                    boolean useSnappy = true;
                    try {
                        String compressedList = delimiter == null ? CompressionUtils.compressAndBase64Encode(filteredList, useSnappy) : CompressionUtils.compressAndBase64Encode(filteredList, useSnappy, delimiter);
                        StringBuilder sb = new StringBuilder(3);
                        sb.append(SSE_DATA_PREFIX);
                        sb.append(compressedList);
                        sb.append(TWO_NEWLINES);
                        msgProcessedCounter.increment(valueList.size());
                        lastResponseSent.set(System.currentTimeMillis());
                        return response.writeStringAndFlush(sb.toString());
                    } catch (Exception e) {
                        LOG.warn("Could not compress data" + e.getMessage());
                        droppedWrites.increment(valueList.size());
                        return Observable.empty();
                    }
                } else {
                    int noOfMsgs = 0;
                    StringBuilder sb = new StringBuilder(valueList.size() * 3);
                    for (String s : filteredList) {
                        sb.append(SSE_DATA_PREFIX);
                        sb.append(s);
                        sb.append(TWO_NEWLINES);
                        noOfMsgs++;
                    }
                    msgProcessedCounter.increment(noOfMsgs);
                    lastResponseSent.set(System.currentTimeMillis());
                    return response.writeStringAndFlush(sb.toString());
                }
            } else {
                // 
                droppedWrites.increment(filteredList.size());
            }
            return Observable.empty();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<? extends Void>>() {

        @Override
        public Observable<? extends Void> call(Throwable throwable) {
            Throwable cause = throwable.getCause();
            // ignore closed channel exceptions, this is
            // when the connection was closed on the client
            // side without informing the server
            errorCounter.increment();
            if (cause != null && !(cause instanceof ClosedChannelException)) {
                LOG.warn("Error detected in SSE sink", cause);
                if (errorEncoder != null) {
                    // write error out on connection
                    // response.writeAndFlush(errorEncoder.call(throwable));
                    ByteBuf errType = response.getAllocator().buffer().writeBytes("error: ".getBytes());
                    ByteBuf errRes = response.getAllocator().buffer().writeBytes((errorEncoder.call(throwable)).getBytes());
                    response.writeAndFlush(ServerSentEvent.withEventType(errType, errRes));
                }
                throwable.printStackTrace();
            }
            if (requestPostprocessor != null && queryParameters != null) {
                requestPostprocessor.call(queryParameters, context);
            }
            ssm.deregisterServer(sn, queryParameters);
            timerSubscription.unsubscribe();
            return Observable.error(throwable);
        }
    });
}

Example 9 with ServerSentEvent

use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.

the class MetricsServer method start.

public void start() {
    final Observable<Measurements> measurements = measurements(publishRateInSeconds);
    logger.info("Starting metrics server on port: " + port);
    server = RxNetty.createHttpServer(port, new RequestHandler<ByteBuf, ServerSentEvent>() {

        @Override
        public Observable<Void> handle(HttpServerRequest<ByteBuf> request, final HttpServerResponse<ServerSentEvent> response) {
            final Map<String, List<String>> queryParameters = request.getQueryParameters();
            final List<String> namesToFilter = new LinkedList<>();
            logger.info("got query params {}", queryParameters);
            if (queryParameters != null && queryParameters.containsKey("name")) {
                namesToFilter.addAll(queryParameters.get("name"));
            }
            Observable<Measurements> filteredObservable = measurements.filter(new Func1<Measurements, Boolean>() {

                @Override
                public Boolean call(Measurements measurements) {
                    if (!namesToFilter.isEmpty()) {
                        // check filters
                        for (String name : namesToFilter) {
                            if (name.indexOf('*') != -1) {
                                // check for ends with
                                if (name.indexOf('*') == 0 && measurements.getName().endsWith(name.substring(1)))
                                    return true;
                                // check for starts with
                                if (name.indexOf('*') > 0 && measurements.getName().startsWith(name.substring(0, name.indexOf('*')))) {
                                    return true;
                                }
                            }
                            if (measurements.getName().equals(name)) {
                                // filter match
                                return true;
                            }
                        }
                        // not found in filters
                        return false;
                    } else {
                        // no filters provided
                        return true;
                    }
                }
            });
            return filteredObservable.flatMap(new Func1<Measurements, Observable<Void>>() {

                @Override
                public Observable<Void> call(Measurements metrics) {
                    response.getHeaders().set("Access-Control-Allow-Origin", "*");
                    response.getHeaders().set("content-type", "text/event-stream");
                    ServerSentEvent event = null;
                    try {
                        ByteBuf data = response.getAllocator().buffer().writeBytes((mapper.writeValueAsString(metrics)).getBytes());
                        event = new ServerSentEvent(data);
                    // event = new ServerSentEvent(null, "data", mapper.writeValueAsString(metrics));
                    } catch (JsonProcessingException e) {
                        logger.error("Failed to map metrics to JSON", e);
                    }
                    if (event != null) {
                        response.write(event);
                        return response.writeStringAndFlush("\n");
                    }
                    return null;
                }
            });
        }
    }, PipelineConfigurators.<ByteBuf>serveSseConfigurator()).start();
}

Example 10 with ServerSentEvent

use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.

the class HttpSourceImplTest method testGettingStreamFromMultipleServers.

@Test
public void testGettingStreamFromMultipleServers() throws Exception {
    HttpSourceImpl<ByteBuf, ServerSentEvent, ServerContext<ServerSentEvent>> source = createStreamingSource();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {

        @Override
        public void call(ServerContext<ServerSentEvent> pair) {
            assertTrue(pair.getValue().contentAsString().contains("line"));
            counter.incrementAndGet();
            String msg = pair.getValue().contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (Server server : localServerProvider.getServers()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(toServerInfo(server), EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
    assertEquals("completed source should clean up its retry servers", 0, source.getRetryServers().size());
}