use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.
the class TestSseServerFactory method newServerWithInitialData.
public static int newServerWithInitialData(final int port, final String data) {
final HttpServer<String, ServerSentEvent> server = RxNetty.newHttpServerBuilder(port, new RequestHandler<String, ServerSentEvent>() {
@Override
public Observable<Void> handle(HttpServerRequest<String> req, HttpServerResponse<ServerSentEvent> resp) {
final ByteBuf byteBuf = resp.getAllocator().buffer().writeBytes(data.getBytes());
resp.writeAndFlush(new ServerSentEvent(byteBuf));
return Observable.empty();
}
}).pipelineConfigurator(PipelineConfigurators.<String>serveSseConfigurator()).channelOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(1024 * 1024, 5 * 1024 * 1024)).build();
server.start();
synchronized (servers) {
servers.add(server);
}
return port;
}
use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.
the class HttpSourceTest method sourceEchoStreamFromPost.
private void sourceEchoStreamFromPost(HttpSource<ServerSentEvent, ServerSentEvent> source, String postContent) throws Exception {
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {
@Override
public void call(ServerSentEvent event) {
counter.incrementAndGet();
String msg = event.contentAsString();
result.putIfAbsent(msg, new AtomicInteger());
result.get(msg).incrementAndGet();
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
done.countDown();
}
}).subscribe();
long waitSeconds = 30000;
boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
if (timedout) {
fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
}
Assert.assertEquals(String.format("%d servers => the result has %d times of a single echo", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), counter.get());
assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(postContent).get());
for (ServerInfo server : localServerProvider.getServerInfos()) {
assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
}
assertEquals(1, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertEquals(EXPECTED_EVENTS_SETS, events);
for (EventType event : events) {
assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
}
}
use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.
the class SseWorkerConnectionTest method testStreamContentDrops.
@Test
public void testStreamContentDrops() throws Exception {
SpectatorRegistryFactory.setRegistry(new DefaultRegistry());
String metricGroupString = "testmetric";
MetricGroupId metricGroupId = new MetricGroupId(metricGroupString);
SseWorkerConnection workerConnection = new SseWorkerConnection("connection_type", "hostname", 80, b -> {
}, b -> {
}, t -> {
}, 600, false, new CopyOnWriteArraySet<>(), 1, null, true, metricGroupId);
HttpClientResponse<ServerSentEvent> response = mock(HttpClientResponse.class);
TestScheduler testScheduler = Schedulers.test();
// Events are just "0", "1", "2", ...
Observable<ServerSentEvent> contentObs = Observable.interval(1, TimeUnit.SECONDS, testScheduler).map(t -> new ServerSentEvent(Unpooled.copiedBuffer(Long.toString(t), Charset.defaultCharset())));
when(response.getContent()).thenReturn(contentObs);
TestSubscriber<MantisServerSentEvent> subscriber = new TestSubscriber<>(1);
workerConnection.streamContent(response, b -> {
}, 600, "delimiter").subscribeOn(testScheduler).subscribe(subscriber);
testScheduler.advanceTimeBy(100, TimeUnit.SECONDS);
subscriber.assertValueCount(1);
List<MantisServerSentEvent> events = subscriber.getOnNextEvents();
assertEquals("0", events.get(0).getEventAsString());
Metrics metrics = MetricsRegistry.getInstance().getMetric(metricGroupId);
Counter onNextCounter = metrics.getCounter(DropOperator.Counters.onNext.toString());
Counter droppedCounter = metrics.getCounter(DropOperator.Counters.dropped.toString());
logger.info("next: {}", onNextCounter.value());
logger.info("drop: {}", droppedCounter.value());
assertTrue(onNextCounter.value() < 10);
assertTrue(droppedCounter.value() > 90);
}
use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.
the class PushServerSse method createServer.
@Override
public RxServer<?, ?> createServer() {
RxServer<HttpServerRequest<String>, HttpServerResponse<ServerSentEvent>> server = RxNetty.newHttpServerBuilder(port, new RequestHandler<String, ServerSentEvent>() {
@Override
public Observable<Void> handle(HttpServerRequest<String> request, final HttpServerResponse<ServerSentEvent> response) {
final Map<String, List<String>> queryParameters = request.getQueryParameters();
final Counter sseProcessedCounter;
final Counter sseDroppedCounter;
// heartbeat state
boolean enableHeartbeats = false;
boolean enableBinaryOutput = false;
final AtomicLong heartBeatReadIdleSec = new AtomicLong(2);
SerializedSubject<String, String> metaMsgSubject = PublishSubject.<String>create().toSerialized();
final AtomicLong metaMessagesFreqMSec = new AtomicLong(1000);
boolean enableMetaMessages = false;
final AtomicLong lastWriteTime = new AtomicLong();
Subscription heartbeatSubscription = null;
Subscription metaMsgSubscription = null;
// sample state
boolean enableSampling = false;
long samplingTimeMsec = 0;
// client state
String groupId = null;
String slotId = null;
String id = null;
Func1<T, Boolean> predicateFunction = null;
if (predicate != null) {
predicateFunction = predicate.call(queryParameters);
}
byte[] delimiter = CompressionUtils.MANTIS_SSE_DELIMITER_BINARY;
if (queryParameters != null && !queryParameters.isEmpty()) {
if (queryParameters.containsKey(MantisSSEConstants.ID)) {
id = queryParameters.get(MantisSSEConstants.ID).get(0);
}
if (queryParameters.containsKey(MantisSSEConstants.SLOT_ID)) {
slotId = queryParameters.get(MantisSSEConstants.SLOT_ID).get(0);
}
// support groupId and clientId for grouping
if (queryParameters.containsKey(MantisSSEConstants.GROUP_ID)) {
groupId = queryParameters.get(MantisSSEConstants.GROUP_ID).get(0);
}
if (queryParameters.containsKey(MantisSSEConstants.CLIENT_ID)) {
groupId = queryParameters.get(MantisSSEConstants.CLIENT_ID).get(0);
}
if (queryParameters.containsKey(MantisSSEConstants.HEARTBEAT_SEC)) {
heartBeatReadIdleSec.set(Long.parseLong(queryParameters.get(MantisSSEConstants.HEARTBEAT_SEC).get(0)));
if (heartBeatReadIdleSec.get() < 1) {
throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
}
enableHeartbeats = true;
}
if (queryParameters != null && queryParameters.containsKey(MantisSSEConstants.MANTIS_ENABLE_COMPRESSION)) {
String enableBinaryOutputStr = queryParameters.get(MantisSSEConstants.MANTIS_ENABLE_COMPRESSION).get(0);
if ("true".equalsIgnoreCase(enableBinaryOutputStr)) {
logger.info("Binary compression requested");
enableBinaryOutput = true;
}
}
if (queryParameters.containsKey(MantisSSEConstants.ENABLE_PINGS)) {
String enablePings = queryParameters.get(MantisSSEConstants.ENABLE_PINGS).get(0);
if ("true".equalsIgnoreCase(enablePings)) {
enableHeartbeats = true;
}
}
if (queryParameters.containsKey(MantisSSEConstants.ENABLE_META_MESSAGES)) {
String enableMetaMessagesStr = queryParameters.get(MantisSSEConstants.ENABLE_META_MESSAGES).get(0);
if ("true".equalsIgnoreCase(enableMetaMessagesStr)) {
enableMetaMessages = true;
}
}
if (queryParameters.containsKey(MantisSSEConstants.META_MESSAGES_SEC)) {
metaMessagesFreqMSec.set(Long.parseLong(queryParameters.get(MantisSSEConstants.META_MESSAGES_SEC).get(0)));
if (metaMessagesFreqMSec.get() < 250) {
throw new IllegalArgumentException("Meta message frequence rate too low: " + metaMessagesFreqMSec.get());
}
enableMetaMessages = true;
}
if (queryParameters.containsKey(MantisSSEConstants.SAMPLE)) {
samplingTimeMsec = Long.parseLong(queryParameters.get(MantisSSEConstants.SAMPLE).get(0)) * 1000;
if (samplingTimeMsec < 50) {
throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
}
enableSampling = true;
}
if (queryParameters.containsKey(MantisSSEConstants.SAMPLE_M_SEC)) {
samplingTimeMsec = Long.parseLong(queryParameters.get(MantisSSEConstants.SAMPLE_M_SEC).get(0));
if (samplingTimeMsec < 50) {
throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
}
enableSampling = true;
}
if (queryParameters.containsKey(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER)) {
String rawDelimiter = queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0);
if (rawDelimiter != null && !rawDelimiter.isEmpty()) {
delimiter = rawDelimiter.getBytes();
}
}
if (queryParameters.containsKey(MantisSSEConstants.MQL)) {
String query = queryParameters.get(MantisSSEConstants.MQL).get(0);
if ((Boolean) mqlParses.invoke(query)) {
Query q = (Query) mqlMakeQuery.invoke(groupId, query);
predicateFunction = (T datum) -> datum instanceof Map ? q.matches((Map) datum) : true;
}
}
}
InetSocketAddress socketAddress = (InetSocketAddress) response.getChannel().remoteAddress();
Metrics metrics;
if (groupId == null) {
String address = socketAddress.getAddress().toString();
metrics = registerSseMetrics(address, address);
} else {
metrics = registerSseMetrics(groupId, socketAddress.getAddress().toString());
}
sseProcessedCounter = metrics.getCounter(PROCESSED_COUNTER_METRIC_NAME);
sseDroppedCounter = metrics.getCounter(DROPPED_COUNTER_METRIC_NAME);
response.getHeaders().set("Access-Control-Allow-Origin", "*");
response.getHeaders().set("content-type", "text/event-stream");
response.getHeaders().set("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate");
response.getHeaders().set("Pragma", "no-cache");
response.flush();
if (queryParameters != null && requestPreprocessor != null) {
requestPreprocessor.call(queryParameters, processorState);
}
if (enableMetaMessages && metaMessagesFreqMSec.get() > 0) {
logger.info("Enabling Meta messages, interval : " + metaMessagesFreqMSec.get() + " ms");
metaMsgSubscription = metaMsgSubject.throttleLast(metaMessagesFreqMSec.get(), TimeUnit.MILLISECONDS).doOnNext((String t) -> {
if (t != null && !t.isEmpty()) {
long currentTime = System.currentTimeMillis();
ByteBuf data = response.getAllocator().buffer().writeBytes(t.getBytes());
response.writeAndFlush(new ServerSentEvent(data));
lastWriteTime.set(currentTime);
}
}).subscribe();
}
if (enableHeartbeats && heartBeatReadIdleSec.get() > 0) {
logger.info("Enabling hearts, interval: " + heartBeatReadIdleSec);
heartbeatSubscription = Observable.interval(2, heartBeatReadIdleSec.get(), TimeUnit.SECONDS).doOnNext((Long t1) -> {
long currentTime = System.currentTimeMillis();
long diff = (currentTime - lastWriteTime.get()) / 1000;
if (diff > heartBeatReadIdleSec.get()) {
ByteBuf data = response.getAllocator().buffer().writeBytes("ping".getBytes());
response.writeAndFlush(new ServerSentEvent(data));
lastWriteTime.set(currentTime);
}
}).subscribe();
}
Action0 connectionClosedCallback = null;
if (queryParameters != null && requestPostprocessor != null) {
connectionClosedCallback = new Action0() {
@Override
public void call() {
requestPostprocessor.call(queryParameters, processorState);
}
};
}
class SubscribeCallback implements Action0 {
@Override
public void call() {
if (queryParameters != null && subscribeProcessor != null) {
subscribeProcessor.call(queryParameters, processorState);
}
}
}
return manageConnectionWithCompression(response, socketAddress.getHostString(), socketAddress.getPort(), groupId, slotId, id, lastWriteTime, enableHeartbeats, heartbeatSubscription, enableSampling, samplingTimeMsec, metaMsgSubject, metaMsgSubscription, predicateFunction, connectionClosedCallback, sseProcessedCounter, sseDroppedCounter, new SubscribeCallback(), enableBinaryOutput, true, delimiter);
}
}).pipelineConfigurator(PipelineConfigurators.serveSseConfigurator()).channelOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(1024 * 1024, 5 * 1024 * 1024)).build();
return server;
}
use of mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent in project mantis by Netflix.
the class ContextualHttpSourceTest method canStreamFromMultipleServersWithCorrectContext.
@Test
public void canStreamFromMultipleServersWithCorrectContext() throws Exception {
ContextualHttpSource<ServerSentEvent> source = HttpSources.contextualSource(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
final CopyOnWriteArraySet<ServerInfo> connectedServers = new CopyOnWriteArraySet<>();
Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {
@Override
public void call(ServerContext<ServerSentEvent> pair) {
assertTrue(pair.getValue().contentAsString().contains("line"));
counter.incrementAndGet();
String msg = pair.getValue().contentAsString();
result.putIfAbsent(msg, new AtomicInteger());
result.get(msg).incrementAndGet();
connectedServers.add(pair.getServer());
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doOnCompleted(new Action0() {
@Override
public void call() {
System.out.println("completed");
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
done.countDown();
}
}).subscribe();
long waitSeconds = 3;
boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
if (timedout) {
fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
}
assertEquals("There should be as many as provided servers", localServerProvider.serverSize(), connectedServers.size());
Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
for (String data : RequestProcessor.smallStreamContent) {
assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
}
for (ServerInfo server : localServerProvider.getServerInfos()) {
assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
}
assertEquals("There should be one completions", 1, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertEquals(EXPECTED_EVENTS_SETS, events);
for (EventType event : events) {
assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
}
}
Aggregations