use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceTest method pollingSourceWillWork.
@Test
public void pollingSourceWillWork() throws Exception {
ServerInfo server = localServerProvider.getServerInfos().get(0);
HttpSource<ByteBuf, String> source = HttpSources.pollingSource(server.getHost(), server.getPort(), "test/singleEntity").withActivityObserver(sourceObserver).build();
final AtomicInteger counter = new AtomicInteger();
final int maxRepeat = 10;
final CountDownLatch done = new CountDownLatch(maxRepeat);
final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
Subscription subscription = Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<String>() {
@Override
public void call(String content) {
assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, content);
counter.incrementAndGet();
result.putIfAbsent(content, new AtomicInteger());
result.get(content).incrementAndGet();
done.countDown();
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).subscribe();
long waitSeconds = 3;
boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
if (timedout) {
fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
}
Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), maxRepeat);
assertTrue(String.format("There should be at least %d completions after %d repeats (The last one may not have completion. Actual completion count: %d", maxRepeat - 1, maxRepeat, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED)), maxRepeat - 1 <= sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
assertEquals("There should be " + maxRepeat + " connection establishment in total", maxRepeat, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
assertEquals("There should no final completion", 0, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertTrue("Polling Source always has subscriptions, so there won't be subscription_ended event. But other events should all be there", EXPECTED_EVENTS_SETS.containsAll(events));
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceTest method sourcingStream.
private void sourcingStream(HttpSource<ServerSentEvent, ServerSentEvent> source) throws InterruptedException {
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {
@Override
public void call(ServerSentEvent event) {
counter.incrementAndGet();
String msg = event.contentAsString();
result.putIfAbsent(msg, new AtomicInteger());
result.get(msg).incrementAndGet();
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
done.countDown();
}
}).subscribe();
long waitSeconds = 30000;
boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
if (timedout) {
fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
}
Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
for (String data : RequestProcessor.smallStreamContent) {
assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
}
for (ServerInfo server : localServerProvider.getServerInfos()) {
assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
}
assertEquals(1, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertEquals(EXPECTED_EVENTS_SETS, events);
for (EventType event : events) {
assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
}
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceTest method testResubscribeShouldAlwaysWork.
@Test
public void testResubscribeShouldAlwaysWork() throws Exception {
HttpSource<ServerSentEvent, ServerSentEvent> source = HttpSources.source(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
int totalCount = 5;
final CountDownLatch latch = new CountDownLatch(totalCount);
Observable<ServerSentEvent> stream = Observable.merge(source.call(new Context(), new Index(1, 1)));
Subscription sub = stream.subscribe(new Action1<ServerSentEvent>() {
@Override
public void call(ServerSentEvent event) {
latch.countDown();
}
});
long waitSeconds = 10;
boolean countedDown = latch.await(waitSeconds, TimeUnit.SECONDS);
if (!countedDown) {
fail(String.format("Waited too long to receive %d events within %d seconds. Total counted: %d", totalCount, waitSeconds, latch.getCount()));
}
sub.unsubscribe();
final CountDownLatch newLatch = new CountDownLatch(totalCount);
sub = stream.subscribe(new Action1<ServerSentEvent>() {
@Override
public void call(ServerSentEvent event) {
newLatch.countDown();
}
});
countedDown = newLatch.await(5, TimeUnit.SECONDS);
if (!countedDown) {
fail("Waited too long to receive enough events. Counted: " + latch.getCount());
}
sub.unsubscribe();
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class ServerSentEventRequestHandler method handle.
@Override
public Observable<Void> handle(HttpServerRequest<ByteBuf> request, final HttpServerResponse<ServerSentEvent> response) {
InetSocketAddress socketAddress = (InetSocketAddress) response.getChannel().remoteAddress();
LOG.info("HTTP SSE connection received from " + socketAddress.getAddress() + ":" + socketAddress.getPort() + " queryParams: " + request.getQueryParameters());
final String socketAddrStr = socketAddress.getAddress().toString();
final WritableEndpoint<String> sn = new WritableEndpoint<>(socketAddress.getHostString(), socketAddress.getPort(), Endpoint.uniqueHost(socketAddress.getHostString(), socketAddress.getPort(), null));
final Map<String, List<String>> queryParameters = request.getQueryParameters();
final SlotAssignmentManager<String> slotMgr = ssm.registerServer(sn, queryParameters);
final AtomicLong lastResponseFlush = new AtomicLong();
lastResponseFlush.set(-1);
final AtomicLong lastResponseSent = new AtomicLong(-1);
// copy reference, then apply request specific filters, sampling
Observable<T> requestObservable = observableToServe;
// decouple the observable on a separate thread and add backpressure handling
// ServiceRegistry.INSTANCE.getPropertiesService().getStringValue("sse.decouple", "false");
String decoupleSSE = "false";
if ("true".equals(decoupleSSE)) {
final BasicTag sockAddrTag = new BasicTag("sockAddr", Optional.ofNullable(socketAddrStr).orElse("none"));
requestObservable = requestObservable.lift(new DropOperator<T>("outgoing_ServerSentEventRequestHandler", sockAddrTag)).observeOn(Schedulers.io());
}
response.getHeaders().set("Access-Control-Allow-Origin", "*");
response.getHeaders().set("content-type", "text/event-stream");
response.getHeaders().set("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate");
response.getHeaders().set("Pragma", "no-cache");
response.flush();
String uniqueClientId = socketAddrStr;
if (queryParameters != null && queryParameters.containsKey(CLIENT_ID_PARAM)) {
// enablePings
uniqueClientId = queryParameters.get(CLIENT_ID_PARAM).get(0);
}
if (queryParameters != null && queryParameters.containsKey(FORMAT_PARAM)) {
format = queryParameters.get(FORMAT_PARAM).get(0);
}
if (queryParameters != null && requestPreprocessor != null) {
requestPreprocessor.call(queryParameters, context);
}
// apply sampling, milli, then seconds
if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM_MSEC)) {
// apply sampling rate
int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM_MSEC).get(0));
requestObservable = requestObservable.sample(samplingRate, TimeUnit.MILLISECONDS);
}
if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM)) {
// apply sampling rate
int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM).get(0));
requestObservable = requestObservable.sample(samplingRate, TimeUnit.SECONDS);
}
if (queryParameters != null && queryParameters.containsKey(ENABLE_PINGS_PARAM)) {
// enablePings
String enablePings = queryParameters.get(ENABLE_PINGS_PARAM).get(0);
if ("true".equalsIgnoreCase(enablePings)) {
pingsEnabled = true;
} else {
pingsEnabled = false;
}
}
if (queryParameters != null && queryParameters.containsKey("delay")) {
// apply flush
try {
int flushInterval = Integer.parseInt(queryParameters.get("delay").get(0));
if (flushInterval >= 50) {
flushIntervalMillis = flushInterval;
} else {
LOG.warn("delay parameter too small " + flushInterval + " min. is 100");
}
} catch (Exception e) {
e.printStackTrace();
}
}
final byte[] delimiter = queryParameters != null && queryParameters.containsKey(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER) && queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0) != null ? queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0).getBytes() : null;
Tag[] tags = new Tag[2];
final String clientId = Optional.ofNullable(uniqueClientId).orElse("none");
final String sockAddr = Optional.ofNullable(socketAddrStr).orElse("none");
tags[0] = new BasicTag("clientId", clientId);
tags[1] = new BasicTag("sockAddr", sockAddr);
Metrics sseSinkMetrics = new Metrics.Builder().id("ServerSentEventRequestHandler", tags).addCounter("processedCounter").addCounter("pingCounter").addCounter("errorCounter").addCounter("droppedCounter").addCounter("flushCounter").build();
final Counter msgProcessedCounter = sseSinkMetrics.getCounter("processedCounter");
final Counter pingCounter = sseSinkMetrics.getCounter("pingCounter");
final Counter errorCounter = sseSinkMetrics.getCounter("errorCounter");
final Counter droppedWrites = sseSinkMetrics.getCounter("droppedCounter");
final Counter flushCounter = sseSinkMetrics.getCounter("flushCounter");
// get predicate, defaults to return true for all T
Func1<T, Boolean> filterFunction = new Func1<T, Boolean>() {
@Override
public Boolean call(T t1) {
return true;
}
};
if (queryParameters != null && predicate != null) {
filterFunction = predicate.getPredicate().call(queryParameters);
}
final Subscription timerSubscription = Observable.interval(1, TimeUnit.SECONDS).doOnNext(new Action1<Long>() {
@Override
public void call(Long t1) {
long currentTime = System.currentTimeMillis();
if (pingsEnabled && (lastResponseSent.get() == -1 || currentTime > lastResponseSent.get() + PING_INTERVAL)) {
pingCounter.increment();
response.writeStringAndFlush(PING);
lastResponseSent.set(currentTime);
}
}
}).subscribe();
return requestObservable.filter(filterFunction).map(encoder).lift(new DisableBackPressureOperator<String>()).buffer(flushIntervalMillis, TimeUnit.MILLISECONDS).flatMap(new Func1<List<String>, Observable<Void>>() {
@Override
public Observable<Void> call(List<String> valueList) {
if (response.isCloseIssued() || !response.getChannel().isActive()) {
LOG.info("Client closed detected, throwing closed channel exception");
return Observable.error(new ClosedChannelException());
}
List<String> filteredList = valueList.stream().filter(e -> {
return slotMgr.filter(sn, e.getBytes());
}).collect(Collectors.toList());
if (response.getChannel().isWritable()) {
flushCounter.increment();
if (format.equals(BINARY_FORMAT)) {
boolean useSnappy = true;
try {
String compressedList = delimiter == null ? CompressionUtils.compressAndBase64Encode(filteredList, useSnappy) : CompressionUtils.compressAndBase64Encode(filteredList, useSnappy, delimiter);
StringBuilder sb = new StringBuilder(3);
sb.append(SSE_DATA_PREFIX);
sb.append(compressedList);
sb.append(TWO_NEWLINES);
msgProcessedCounter.increment(valueList.size());
lastResponseSent.set(System.currentTimeMillis());
return response.writeStringAndFlush(sb.toString());
} catch (Exception e) {
LOG.warn("Could not compress data" + e.getMessage());
droppedWrites.increment(valueList.size());
return Observable.empty();
}
} else {
int noOfMsgs = 0;
StringBuilder sb = new StringBuilder(valueList.size() * 3);
for (String s : filteredList) {
sb.append(SSE_DATA_PREFIX);
sb.append(s);
sb.append(TWO_NEWLINES);
noOfMsgs++;
}
msgProcessedCounter.increment(noOfMsgs);
lastResponseSent.set(System.currentTimeMillis());
return response.writeStringAndFlush(sb.toString());
}
} else {
//
droppedWrites.increment(filteredList.size());
}
return Observable.empty();
}
}).onErrorResumeNext(new Func1<Throwable, Observable<? extends Void>>() {
@Override
public Observable<? extends Void> call(Throwable throwable) {
Throwable cause = throwable.getCause();
// ignore closed channel exceptions, this is
// when the connection was closed on the client
// side without informing the server
errorCounter.increment();
if (cause != null && !(cause instanceof ClosedChannelException)) {
LOG.warn("Error detected in SSE sink", cause);
if (errorEncoder != null) {
// write error out on connection
// response.writeAndFlush(errorEncoder.call(throwable));
ByteBuf errType = response.getAllocator().buffer().writeBytes("error: ".getBytes());
ByteBuf errRes = response.getAllocator().buffer().writeBytes((errorEncoder.call(throwable)).getBytes());
response.writeAndFlush(ServerSentEvent.withEventType(errType, errRes));
}
throwable.printStackTrace();
}
if (requestPostprocessor != null && queryParameters != null) {
requestPostprocessor.call(queryParameters, context);
}
ssm.deregisterServer(sn, queryParameters);
timerSubscription.unsubscribe();
return Observable.error(throwable);
}
});
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class StageExecutorsTest method testExecuteSource.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSource() {
TestJob provider = new TestJob();
Job<Integer> job = provider.getJobInstance();
List<StageConfig<?, ?>> stages = job.getStages();
PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
int serverPort = portSelector.acquirePort();
WorkerPublisher producer = new WorkerPublisherRemoteObservable(serverPort, null, Observable.just(1), null);
// execute source
BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(1);
StageExecutors.executeSource(0, job.getSource(), stages.get(0), producer, new Context(), workersInStageOneObservable);
Iterator<Integer> iter = RemoteObservable.connect(new ConnectToObservable.Builder<Integer>().host("localhost").slotId("0").port(serverPort).decoder(Codecs.integer()).build()).getObservable().toBlocking().getIterator();
// verify numbers are doubled
Assert.assertEquals(0, iter.next().intValue());
Assert.assertEquals(1, iter.next().intValue());
Assert.assertEquals(4, iter.next().intValue());
Assert.assertEquals(9, iter.next().intValue());
}
Aggregations