Examples with Index io.mantisrx.runtime.source.Index used on opensource projects

Example 6 with Index

use of io.mantisrx.runtime.source.Index in project mantis by Netflix.

the class TestJobParameterized method getJobInstance.

@Override
public Job<Integer> getJobInstance() {
    return MantisJob.<Integer>source(new Source<Integer>() {

        @Override
        public Observable<Observable<Integer>> call(Context context, Index index) {
            Integer start = (Integer) context.getParameters().get("start-range");
            Integer end = (Integer) context.getParameters().get("end-range");
            return Observable.just(Observable.range(start, end));
        }
    }).stage(new ScalarComputation<Integer, Integer>() {

        @Override
        public Observable<Integer> call(Context context, Observable<Integer> t1) {
            final Integer scale = (Integer) context.getParameters().get("scale-by");
            return t1.map(new Func1<Integer, Integer>() {

                @Override
                public Integer call(Integer t1) {
                    return t1 * scale;
                }
            });
        }
    }, new ScalarToScalar.Config<Integer, Integer>().codec(Codecs.integer())).sink(new Sink<Integer>() {

        @Override
        public void init(Context context) {
            System.out.println("sink init called");
        }

        @Override
        public void call(Context context, PortRequest p, Observable<Integer> o) {
            final String message = (String) context.getParameters().get("sink-message");
            o.toBlocking().forEach(new Action1<Integer>() {

                @Override
                public void call(Integer t1) {
                    System.out.println(message + t1);
                    itemsWritten.add(t1);
                }
            });
        }
    }).metadata(new Metadata.Builder().name("test job").description("showcase parameters").build()).parameterDefinition(new IntParameter().name("start-range").validator(Validators.range(0, 100)).build()).parameterDefinition(new IntParameter().name("end-range").validator(Validators.range(100, 1000)).build()).parameterDefinition(new IntParameter().name("scale-by").description("scale each value from the range").validator(Validators.range(1, 10)).build()).parameterDefinition(new StringParameter().name("sink-message").defaultValue("hello: ").description("concat with results").validator(Validators.notNullOrEmpty()).build()).create();
}

Example 7 with Index

use of io.mantisrx.runtime.source.Index in project mantis by Netflix.

the class ContextualHttpSourceTest method canStreamFromMultipleServersWithCorrectContext.

@Test
public void canStreamFromMultipleServersWithCorrectContext() throws Exception {
    ContextualHttpSource<ServerSentEvent> source = HttpSources.contextualSource(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    final CopyOnWriteArraySet<ServerInfo> connectedServers = new CopyOnWriteArraySet<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {

        @Override
        public void call(ServerContext<ServerSentEvent> pair) {
            assertTrue(pair.getValue().contentAsString().contains("line"));
            counter.incrementAndGet();
            String msg = pair.getValue().contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
            connectedServers.add(pair.getServer());
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            System.out.println("completed");
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    assertEquals("There should be as many as provided servers", localServerProvider.serverSize(), connectedServers.size());
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals("There should be one completions", 1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}

Example 8 with Index

use of io.mantisrx.runtime.source.Index in project mantis by Netflix.

the class HttpSourceTest method pollingSourceWillWork.

@Test
public void pollingSourceWillWork() throws Exception {
    ServerInfo server = localServerProvider.getServerInfos().get(0);
    HttpSource<ByteBuf, String> source = HttpSources.pollingSource(server.getHost(), server.getPort(), "test/singleEntity").withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final int maxRepeat = 10;
    final CountDownLatch done = new CountDownLatch(maxRepeat);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Subscription subscription = Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<String>() {

        @Override
        public void call(String content) {
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, content);
            counter.incrementAndGet();
            result.putIfAbsent(content, new AtomicInteger());
            result.get(content).incrementAndGet();
            done.countDown();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), maxRepeat);
    assertTrue(String.format("There should be at least %d completions after %d repeats (The last one may not have completion. Actual completion count: %d", maxRepeat - 1, maxRepeat, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED)), maxRepeat - 1 <= sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
    assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
    assertEquals("There should be " + maxRepeat + " connection establishment in total", maxRepeat, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    assertEquals("There should no final completion", 0, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertTrue("Polling Source always has subscriptions, so there won't be subscription_ended event. But other events should all be there", EXPECTED_EVENTS_SETS.containsAll(events));
}

Example 9 with Index

use of io.mantisrx.runtime.source.Index in project mantis by Netflix.

the class HttpSourceTest method sourcingStream.

private void sourcingStream(HttpSource<ServerSentEvent, ServerSentEvent> source) throws InterruptedException {
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            counter.incrementAndGet();
            String msg = event.contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 30000;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}

Example 10 with Index

use of io.mantisrx.runtime.source.Index in project mantis by Netflix.

the class HttpSourceTest method testResubscribeShouldAlwaysWork.

@Test
public void testResubscribeShouldAlwaysWork() throws Exception {
    HttpSource<ServerSentEvent, ServerSentEvent> source = HttpSources.source(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    int totalCount = 5;
    final CountDownLatch latch = new CountDownLatch(totalCount);
    Observable<ServerSentEvent> stream = Observable.merge(source.call(new Context(), new Index(1, 1)));
    Subscription sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            latch.countDown();
        }
    });
    long waitSeconds = 10;
    boolean countedDown = latch.await(waitSeconds, TimeUnit.SECONDS);
    if (!countedDown) {
        fail(String.format("Waited too long to receive %d events within %d seconds. Total counted: %d", totalCount, waitSeconds, latch.getCount()));
    }
    sub.unsubscribe();
    final CountDownLatch newLatch = new CountDownLatch(totalCount);
    sub = stream.subscribe(new Action1<ServerSentEvent>() {

        @Override
        public void call(ServerSentEvent event) {
            newLatch.countDown();
        }
    });
    countedDown = newLatch.await(5, TimeUnit.SECONDS);
    if (!countedDown) {
        fail("Waited too long to receive enough events. Counted: " + latch.getCount());
    }
    sub.unsubscribe();
}