Search in sources :

Example 26 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceTest method pollingSourceWillWork.

@Test
public void pollingSourceWillWork() throws Exception {
    ServerInfo server = localServerProvider.getServerInfos().get(0);
    HttpSource<ByteBuf, String> source = HttpSources.pollingSource(server.getHost(), server.getPort(), "test/singleEntity").withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final int maxRepeat = 10;
    final CountDownLatch done = new CountDownLatch(maxRepeat);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Subscription subscription = Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<String>() {

        @Override
        public void call(String content) {
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, content);
            counter.incrementAndGet();
            result.putIfAbsent(content, new AtomicInteger());
            result.get(content).incrementAndGet();
            done.countDown();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), maxRepeat);
    assertTrue(String.format("There should be at least %d completions after %d repeats (The last one may not have completion. Actual completion count: %d", maxRepeat - 1, maxRepeat, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED)), maxRepeat - 1 <= sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
    assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
    assertEquals("There should be " + maxRepeat + " connection establishment in total", maxRepeat, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    assertEquals("There should no final completion", 0, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertTrue("Polling Source always has subscriptions, so there won't be subscription_ended event. But other events should all be there", EXPECTED_EVENTS_SETS.containsAll(events));
}
Also used : Context(io.mantisrx.runtime.Context) Action1(rx.functions.Action1) EventType(io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType) ServerInfo(mantis.io.reactivex.netty.client.RxClient.ServerInfo) Index(io.mantisrx.runtime.source.Index) ByteBuf(io.netty.buffer.ByteBuf) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Subscription(rx.Subscription) Test(org.junit.Test)

Example 27 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceTest method sourcingStream.

private void sourcingStream(HttpSource<ServerSentEvent, ServerSentEvent> source) throws InterruptedException {
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            counter.incrementAndGet();
            String msg = event.contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 30000;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}
Also used : Context(io.mantisrx.runtime.Context) Action0(rx.functions.Action0) Action1(rx.functions.Action1) EventType(io.mantisrx.runtime.source.http.impl.HttpSourceImpl.HttpSourceEvent.EventType) ServerInfo(mantis.io.reactivex.netty.client.RxClient.ServerInfo) ServerSentEvent(mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent) Index(io.mantisrx.runtime.source.Index) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 28 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceTest method testResubscribeShouldAlwaysWork.

@Test
public void testResubscribeShouldAlwaysWork() throws Exception {
    HttpSource<ServerSentEvent, ServerSentEvent> source = HttpSources.source(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    int totalCount = 5;
    final CountDownLatch latch = new CountDownLatch(totalCount);
    Observable<ServerSentEvent> stream = Observable.merge(source.call(new Context(), new Index(1, 1)));
    Subscription sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            latch.countDown();
        }
    });
    long waitSeconds = 10;
    boolean countedDown = latch.await(waitSeconds, TimeUnit.SECONDS);
    if (!countedDown) {
        fail(String.format("Waited too long to receive %d events within %d seconds. Total counted: %d", totalCount, waitSeconds, latch.getCount()));
    }
    sub.unsubscribe();
    final CountDownLatch newLatch = new CountDownLatch(totalCount);
    sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            newLatch.countDown();
        }
    });
    countedDown = newLatch.await(5, TimeUnit.SECONDS);
    if (!countedDown) {
        fail("Waited too long to receive enough events. Counted: " + latch.getCount());
    }
    sub.unsubscribe();
}
Also used : Context(io.mantisrx.runtime.Context) Action1(rx.functions.Action1) ServerSentEvent(mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent) Index(io.mantisrx.runtime.source.Index) CountDownLatch(java.util.concurrent.CountDownLatch) Subscription(rx.Subscription) Test(org.junit.Test)

Example 29 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class ServerSentEventRequestHandler method handle.

@Override
public Observable<Void> handle(HttpServerRequest<ByteBuf> request, final HttpServerResponse<ServerSentEvent> response) {
    InetSocketAddress socketAddress = (InetSocketAddress) response.getChannel().remoteAddress();
    LOG.info("HTTP SSE connection received from " + socketAddress.getAddress() + ":" + socketAddress.getPort() + "  queryParams: " + request.getQueryParameters());
    final String socketAddrStr = socketAddress.getAddress().toString();
    final WritableEndpoint<String> sn = new WritableEndpoint<>(socketAddress.getHostString(), socketAddress.getPort(), Endpoint.uniqueHost(socketAddress.getHostString(), socketAddress.getPort(), null));
    final Map<String, List<String>> queryParameters = request.getQueryParameters();
    final SlotAssignmentManager<String> slotMgr = ssm.registerServer(sn, queryParameters);
    final AtomicLong lastResponseFlush = new AtomicLong();
    lastResponseFlush.set(-1);
    final AtomicLong lastResponseSent = new AtomicLong(-1);
    // copy reference, then apply request specific filters, sampling
    Observable<T> requestObservable = observableToServe;
    // decouple the observable on a separate thread and add backpressure handling
    // ServiceRegistry.INSTANCE.getPropertiesService().getStringValue("sse.decouple", "false");
    String decoupleSSE = "false";
    if ("true".equals(decoupleSSE)) {
        final BasicTag sockAddrTag = new BasicTag("sockAddr", Optional.ofNullable(socketAddrStr).orElse("none"));
        requestObservable = requestObservable.lift(new DropOperator<T>("outgoing_ServerSentEventRequestHandler", sockAddrTag)).observeOn(Schedulers.io());
    }
    response.getHeaders().set("Access-Control-Allow-Origin", "*");
    response.getHeaders().set("content-type", "text/event-stream");
    response.getHeaders().set("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate");
    response.getHeaders().set("Pragma", "no-cache");
    response.flush();
    String uniqueClientId = socketAddrStr;
    if (queryParameters != null && queryParameters.containsKey(CLIENT_ID_PARAM)) {
        // enablePings
        uniqueClientId = queryParameters.get(CLIENT_ID_PARAM).get(0);
    }
    if (queryParameters != null && queryParameters.containsKey(FORMAT_PARAM)) {
        format = queryParameters.get(FORMAT_PARAM).get(0);
    }
    if (queryParameters != null && requestPreprocessor != null) {
        requestPreprocessor.call(queryParameters, context);
    }
    // apply sampling, milli, then seconds
    if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM_MSEC)) {
        // apply sampling rate
        int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM_MSEC).get(0));
        requestObservable = requestObservable.sample(samplingRate, TimeUnit.MILLISECONDS);
    }
    if (queryParameters != null && queryParameters.containsKey(SAMPLE_PARAM)) {
        // apply sampling rate
        int samplingRate = Integer.parseInt(queryParameters.get(SAMPLE_PARAM).get(0));
        requestObservable = requestObservable.sample(samplingRate, TimeUnit.SECONDS);
    }
    if (queryParameters != null && queryParameters.containsKey(ENABLE_PINGS_PARAM)) {
        // enablePings
        String enablePings = queryParameters.get(ENABLE_PINGS_PARAM).get(0);
        if ("true".equalsIgnoreCase(enablePings)) {
            pingsEnabled = true;
        } else {
            pingsEnabled = false;
        }
    }
    if (queryParameters != null && queryParameters.containsKey("delay")) {
        // apply flush
        try {
            int flushInterval = Integer.parseInt(queryParameters.get("delay").get(0));
            if (flushInterval >= 50) {
                flushIntervalMillis = flushInterval;
            } else {
                LOG.warn("delay parameter too small " + flushInterval + " min. is 100");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    final byte[] delimiter = queryParameters != null && queryParameters.containsKey(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER) && queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0) != null ? queryParameters.get(MantisSSEConstants.MANTIS_COMPRESSION_DELIMITER).get(0).getBytes() : null;
    Tag[] tags = new Tag[2];
    final String clientId = Optional.ofNullable(uniqueClientId).orElse("none");
    final String sockAddr = Optional.ofNullable(socketAddrStr).orElse("none");
    tags[0] = new BasicTag("clientId", clientId);
    tags[1] = new BasicTag("sockAddr", sockAddr);
    Metrics sseSinkMetrics = new Metrics.Builder().id("ServerSentEventRequestHandler", tags).addCounter("processedCounter").addCounter("pingCounter").addCounter("errorCounter").addCounter("droppedCounter").addCounter("flushCounter").build();
    final Counter msgProcessedCounter = sseSinkMetrics.getCounter("processedCounter");
    final Counter pingCounter = sseSinkMetrics.getCounter("pingCounter");
    final Counter errorCounter = sseSinkMetrics.getCounter("errorCounter");
    final Counter droppedWrites = sseSinkMetrics.getCounter("droppedCounter");
    final Counter flushCounter = sseSinkMetrics.getCounter("flushCounter");
    // get predicate, defaults to return true for all T
    Func1<T, Boolean> filterFunction = new Func1<T, Boolean>() {

        @Override
        public Boolean call(T t1) {
            return true;
        }
    };
    if (queryParameters != null && predicate != null) {
        filterFunction = predicate.getPredicate().call(queryParameters);
    }
    final Subscription timerSubscription = Observable.interval(1, TimeUnit.SECONDS).doOnNext(new Action1<Long>() {

        @Override
        public void call(Long t1) {
            long currentTime = System.currentTimeMillis();
            if (pingsEnabled && (lastResponseSent.get() == -1 || currentTime > lastResponseSent.get() + PING_INTERVAL)) {
                pingCounter.increment();
                response.writeStringAndFlush(PING);
                lastResponseSent.set(currentTime);
            }
        }
    }).subscribe();
    return requestObservable.filter(filterFunction).map(encoder).lift(new DisableBackPressureOperator<String>()).buffer(flushIntervalMillis, TimeUnit.MILLISECONDS).flatMap(new Func1<List<String>, Observable<Void>>() {

        @Override
        public Observable<Void> call(List<String> valueList) {
            if (response.isCloseIssued() || !response.getChannel().isActive()) {
                LOG.info("Client closed detected, throwing closed channel exception");
                return Observable.error(new ClosedChannelException());
            }
            List<String> filteredList = valueList.stream().filter(e -> {
                return slotMgr.filter(sn, e.getBytes());
            }).collect(Collectors.toList());
            if (response.getChannel().isWritable()) {
                flushCounter.increment();
                if (format.equals(BINARY_FORMAT)) {
                    boolean useSnappy = true;
                    try {
                        String compressedList = delimiter == null ? CompressionUtils.compressAndBase64Encode(filteredList, useSnappy) : CompressionUtils.compressAndBase64Encode(filteredList, useSnappy, delimiter);
                        StringBuilder sb = new StringBuilder(3);
                        sb.append(SSE_DATA_PREFIX);
                        sb.append(compressedList);
                        sb.append(TWO_NEWLINES);
                        msgProcessedCounter.increment(valueList.size());
                        lastResponseSent.set(System.currentTimeMillis());
                        return response.writeStringAndFlush(sb.toString());
                    } catch (Exception e) {
                        LOG.warn("Could not compress data" + e.getMessage());
                        droppedWrites.increment(valueList.size());
                        return Observable.empty();
                    }
                } else {
                    int noOfMsgs = 0;
                    StringBuilder sb = new StringBuilder(valueList.size() * 3);
                    for (String s : filteredList) {
                        sb.append(SSE_DATA_PREFIX);
                        sb.append(s);
                        sb.append(TWO_NEWLINES);
                        noOfMsgs++;
                    }
                    msgProcessedCounter.increment(noOfMsgs);
                    lastResponseSent.set(System.currentTimeMillis());
                    return response.writeStringAndFlush(sb.toString());
                }
            } else {
                // 
                droppedWrites.increment(filteredList.size());
            }
            return Observable.empty();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<? extends Void>>() {

        @Override
        public Observable<? extends Void> call(Throwable throwable) {
            Throwable cause = throwable.getCause();
            // ignore closed channel exceptions, this is
            // when the connection was closed on the client
            // side without informing the server
            errorCounter.increment();
            if (cause != null && !(cause instanceof ClosedChannelException)) {
                LOG.warn("Error detected in SSE sink", cause);
                if (errorEncoder != null) {
                    // write error out on connection
                    // response.writeAndFlush(errorEncoder.call(throwable));
                    ByteBuf errType = response.getAllocator().buffer().writeBytes("error: ".getBytes());
                    ByteBuf errRes = response.getAllocator().buffer().writeBytes((errorEncoder.call(throwable)).getBytes());
                    response.writeAndFlush(ServerSentEvent.withEventType(errType, errRes));
                }
                throwable.printStackTrace();
            }
            if (requestPostprocessor != null && queryParameters != null) {
                requestPostprocessor.call(queryParameters, context);
            }
            ssm.deregisterServer(sn, queryParameters);
            timerSubscription.unsubscribe();
            return Observable.error(throwable);
        }
    });
}
Also used : Predicate(io.mantisrx.runtime.sink.predicate.Predicate) DisableBackPressureOperator(io.reactivx.mantis.operators.DisableBackPressureOperator) ServerSentEvent(mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent) LoggerFactory(org.slf4j.LoggerFactory) Action1(rx.functions.Action1) DropOperator(io.reactivx.mantis.operators.DropOperator) HttpServerResponse(mantis.io.reactivex.netty.protocol.http.server.HttpServerResponse) Observable(rx.Observable) Func1(rx.functions.Func1) Func2(rx.functions.Func2) ByteBuf(io.netty.buffer.ByteBuf) Map(java.util.Map) Schedulers(rx.schedulers.Schedulers) RequestHandler(mantis.io.reactivex.netty.protocol.http.server.RequestHandler) BasicTag(com.netflix.spectator.api.BasicTag) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) Logger(org.slf4j.Logger) Tag(com.netflix.spectator.api.Tag) HashFunctions(io.mantisrx.common.network.HashFunctions) Endpoint(io.mantisrx.common.network.Endpoint) ClosedChannelException(java.nio.channels.ClosedChannelException) WritableEndpoint(io.mantisrx.common.network.WritableEndpoint) Context(io.mantisrx.runtime.Context) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) CompressionUtils(io.mantisrx.common.compression.CompressionUtils) SlotAssignmentManager(io.mantisrx.common.network.ServerSlotManager.SlotAssignmentManager) Optional(java.util.Optional) ServerSlotManager(io.mantisrx.common.network.ServerSlotManager) MantisSSEConstants(com.mantisrx.common.utils.MantisSSEConstants) HttpServerRequest(mantis.io.reactivex.netty.protocol.http.server.HttpServerRequest) Subscription(rx.Subscription) InetSocketAddress(java.net.InetSocketAddress) ByteBuf(io.netty.buffer.ByteBuf) BasicTag(com.netflix.spectator.api.BasicTag) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) List(java.util.List) Func1(rx.functions.Func1) Subscription(rx.Subscription) ClosedChannelException(java.nio.channels.ClosedChannelException) WritableEndpoint(io.mantisrx.common.network.WritableEndpoint) DisableBackPressureOperator(io.reactivx.mantis.operators.DisableBackPressureOperator) Action1(rx.functions.Action1) Endpoint(io.mantisrx.common.network.Endpoint) WritableEndpoint(io.mantisrx.common.network.WritableEndpoint) ClosedChannelException(java.nio.channels.ClosedChannelException) Observable(rx.Observable) AtomicLong(java.util.concurrent.atomic.AtomicLong) AtomicLong(java.util.concurrent.atomic.AtomicLong) BasicTag(com.netflix.spectator.api.BasicTag) Tag(com.netflix.spectator.api.Tag)

Example 30 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsTest method testExecuteSource.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSource() {
    TestJob provider = new TestJob();
    Job<Integer> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    int serverPort = portSelector.acquirePort();
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(serverPort, null, Observable.just(1), null);
    // execute source
    BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(1);
    StageExecutors.executeSource(0, job.getSource(), stages.get(0), producer, new Context(), workersInStageOneObservable);
    Iterator<Integer> iter = RemoteObservable.connect(new ConnectToObservable.Builder<Integer>().host("localhost").slotId("0").port(serverPort).decoder(Codecs.integer()).build()).getObservable().toBlocking().getIterator();
    // verify numbers are doubled
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(1, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
    Assert.assertEquals(9, iter.next().intValue());
}
Also used : Context(io.mantisrx.runtime.Context) PortSelectorWithinRange(io.reactivex.mantis.remote.observable.PortSelectorWithinRange) StageConfig(io.mantisrx.runtime.StageConfig) Endpoint(io.mantisrx.common.network.Endpoint) ConnectToObservable(io.reactivex.mantis.remote.observable.ConnectToObservable) Test(org.junit.Test)

Aggregations

Context (io.mantisrx.runtime.Context)49 Test (org.junit.Test)34 CountDownLatch (java.util.concurrent.CountDownLatch)24 Index (io.mantisrx.runtime.source.Index)23 Observable (rx.Observable)23 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)20 Action1 (rx.functions.Action1)19 Action0 (rx.functions.Action0)17 Map (java.util.Map)16 Parameters (io.mantisrx.runtime.parameter.Parameters)15 Logger (org.slf4j.Logger)13 LoggerFactory (org.slf4j.LoggerFactory)13 ParameterTestUtils (io.mantisrx.connector.kafka.ParameterTestUtils)11 TimeUnit (java.util.concurrent.TimeUnit)11 StageConfig (io.mantisrx.runtime.StageConfig)10 HashMap (java.util.HashMap)10 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)10 Endpoint (io.mantisrx.common.network.Endpoint)9 WorkerInfo (io.mantisrx.runtime.WorkerInfo)9 ByteBuf (io.netty.buffer.ByteBuf)8