use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class WorkerExecutionOperationsNetworkStage method executeStage.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void executeStage(final ExecutionDetails setup) {
ExecuteStageRequest executionRequest = setup.getExecuteStageRequest().getRequest();
// Initialize the schedulingInfo observable for current job and mark it shareable to be reused by anyone interested in this data.
// Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).switchMap((e) -> Observable.just(e).repeatWhen(x -> x.delay(5 , TimeUnit.SECONDS))).subscribeOn(Schedulers.io()).share();
Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).subscribeOn(Schedulers.io()).share();
WorkerInfo workerInfo = generateWorkerInfo(executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getStage(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber(), executionRequest.getDurationType(), "host", executionRequest.getWorkerPorts());
final Observable<Integer> sourceStageTotalWorkersObs = createSourceStageTotalWorkersObservable(selfSchedulingInfo);
RunningWorker.Builder rwBuilder = new RunningWorker.Builder().job(setup.getMantisJob()).schedulingInfo(executionRequest.getSchedulingInfo()).stageTotalWorkersObservable(sourceStageTotalWorkersObs).jobName(executionRequest.getJobName()).stageNum(executionRequest.getStage()).workerIndex(executionRequest.getWorkerIndex()).workerNum(executionRequest.getWorkerNumber()).totalStages(executionRequest.getTotalNumStages()).metricsPort(executionRequest.getMetricsPort()).ports(executionRequest.getPorts().iterator()).jobStatusObserver(setup.getStatus()).requestSubject(setup.getExecuteStageRequest().getRequestSubject()).workerInfo(workerInfo).vmTaskStatusObservable(vmTaskStatusObserver).hasJobMaster(executionRequest.getHasJobMaster()).jobId(executionRequest.getJobId());
if (executionRequest.getStage() == 0) {
rwBuilder = rwBuilder.stage(new JobMasterStageConfig("jobmasterconfig"));
} else {
rwBuilder = rwBuilder.stage((StageConfig) setup.getMantisJob().getStages().get(executionRequest.getStage() - 1));
}
final RunningWorker rw = rwBuilder.build();
AtomicReference<SubscriptionStateHandler> subscriptionStateHandlerRef = new AtomicReference<>();
if (rw.getStageNum() == rw.getTotalStagesNet()) {
// set up subscription state handler only for sink (last) stage
subscriptionStateHandlerRef.set(setupSubscriptionStateHandler(setup.getExecuteStageRequest().getRequest().getJobId(), mantisMasterApi, setup.getExecuteStageRequest().getRequest().getSubscriptionTimeoutSecs(), setup.getExecuteStageRequest().getRequest().getMinRuntimeSecs()));
}
logger.info("Running worker info: " + rw);
rw.signalStartedInitiated();
try {
logger.info(">>>>>>>>>>>>>>>>Calling lifecycle.startup()");
Lifecycle lifecycle = rw.getJob().getLifecycle();
lifecycle.startup();
ServiceLocator serviceLocator = lifecycle.getServiceLocator();
if (lookupSpectatorRegistry) {
try {
final Registry spectatorRegistry = serviceLocator.service(Registry.class);
SpectatorRegistryFactory.setRegistry(spectatorRegistry);
} catch (Throwable t) {
logger.error("failed to init spectator registry using service locator, falling back to {}", SpectatorRegistryFactory.getRegistry().getClass().getCanonicalName());
}
}
// create job context
Parameters parameters = ParameterUtils.createContextParameters(rw.getJob().getParameterDefinitions(), setup.getParameters());
final Context context = generateContext(parameters, serviceLocator, workerInfo, MetricsRegistry.getInstance(), () -> {
rw.signalCompleted();
// wait for completion signal to go to the master and us getting killed. Upon timeout, exit.
try {
Thread.sleep(60000);
} catch (InterruptedException ie) {
logger.warn("Unexpected exception sleeping: " + ie.getMessage());
}
System.exit(0);
}, createWorkerMapObservable(selfSchedulingInfo, executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getDurationType()));
// context.setPrevStageCompletedObservable(createPrevStageCompletedObservable(selfSchedulingInfo, rw.getJobId(), rw.getStageNum()));
rw.setContext(context);
// setup heartbeats
heartbeatRef.set(new Heartbeat(rw.getJobId(), rw.getStageNum(), rw.getWorkerIndex(), rw.getWorkerNum()));
final double networkMbps = executionRequest.getSchedulingInfo().forStage(rw.getStageNum()).getMachineDefinition().getNetworkMbps();
startSendingHeartbeats(rw.getJobStatus(), new WorkerId(executionRequest.getJobId(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber()).getId(), networkMbps);
// execute stage
if (rw.getStageNum() == 0) {
logger.info("JobId: " + rw.getJobId() + ", executing Job Master");
final AutoScaleMetricsConfig autoScaleMetricsConfig = new AutoScaleMetricsConfig();
// Temporary workaround to enable auto-scaling by custom metric in Job Master. This will be revisited to get the entire autoscaling config
// for a job as a System parameter in the JobMaster
final String autoScaleMetricString = (String) parameters.get(JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, "");
if (!Strings.isNullOrEmpty(autoScaleMetricString)) {
final List<String> tokens = Splitter.on("::").omitEmptyStrings().trimResults().splitToList(autoScaleMetricString);
if (tokens.size() == 3) {
final String metricGroup = tokens.get(0);
final String metricName = tokens.get(1);
final String algo = tokens.get(2);
try {
final AutoScaleMetricsConfig.AggregationAlgo aggregationAlgo = AutoScaleMetricsConfig.AggregationAlgo.valueOf(algo);
logger.info("registered UserDefined auto scale metric {}:{} algo {}", metricGroup, metricName, aggregationAlgo);
autoScaleMetricsConfig.addUserDefinedMetric(metricGroup, metricName, aggregationAlgo);
} catch (IllegalArgumentException e) {
final String errorMsg = String.format("ERROR: Invalid algorithm value %s for param %s (algo should be one of %s)", autoScaleMetricsConfig, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, Arrays.stream(AutoScaleMetricsConfig.AggregationAlgo.values()).map(a -> a.name()).collect(Collectors.toList()));
logger.error(errorMsg);
throw new RuntimeException(errorMsg);
}
} else {
final String errorMsg = String.format("ERROR: Invalid value %s for param %s", autoScaleMetricString, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
logger.error(errorMsg);
throw new RuntimeException(errorMsg);
}
} else {
logger.info("param {} is null or empty", JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
}
final JobMasterService jobMasterService = new JobMasterService(rw.getJobId(), rw.getSchedulingInfo(), workerMetricsClient, autoScaleMetricsConfig, mantisMasterApi, rw.getContext(), rw.getOnCompleteCallback(), rw.getOnErrorCallback(), rw.getOnTerminateCallback());
jobMasterService.start();
signalStarted(rw, subscriptionStateHandlerRef);
// block until worker terminates
rw.waitUntilTerminate();
} else if (rw.getStageNum() == 1 && rw.getTotalStagesNet() == 1) {
logger.info("JobId: " + rw.getJobId() + ", single stage job, executing entire job");
// single stage, execute entire job on this machine
PortSelector portSelector = new PortSelector() {
@Override
public int acquirePort() {
return rw.getPorts().next();
}
};
RxMetrics rxMetrics = new RxMetrics();
StageExecutors.executeSingleStageJob(rw.getJob().getSource(), rw.getStage(), rw.getJob().getSink(), portSelector, rxMetrics, rw.getContext(), rw.getOnTerminateCallback(), rw.getWorkerIndex(), rw.getSourceStageTotalWorkersObservable(), onSinkSubscribe, onSinkUnsubscribe, rw.getOnCompleteCallback(), rw.getOnErrorCallback());
signalStarted(rw, subscriptionStateHandlerRef);
// block until worker terminates
rw.waitUntilTerminate();
} else {
logger.info("JobId: " + rw.getJobId() + ", executing a multi-stage job, stage: " + rw.getStageNum());
if (rw.getStageNum() == 1) {
// execute source stage
String remoteObservableName = rw.getJobId() + "_" + rw.getStageNum();
StageSchedulingInfo currentStageSchedulingInfo = rw.getSchedulingInfo().forStage(1);
WorkerPublisherRemoteObservable publisher = new WorkerPublisherRemoteObservable<>(rw.getPorts().next(), remoteObservableName, numWorkersAtStage(selfSchedulingInfo, rw.getJobId(), rw.getStageNum() + 1), rw.getJobName());
StageExecutors.executeSource(rw.getWorkerIndex(), rw.getJob().getSource(), rw.getStage(), publisher, rw.getContext(), rw.getSourceStageTotalWorkersObservable());
logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", serving remote observable for source with name: " + remoteObservableName);
RemoteRxServer server = publisher.getServer();
RxMetrics rxMetrics = server.getMetrics();
MetricsRegistry.getInstance().registerAndGet(rxMetrics.getCountersAndGauges());
signalStarted(rw, subscriptionStateHandlerRef);
logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", blocking until source observable completes");
server.blockUntilServerShutdown();
} else {
// execute intermediate stage or last stage plus sink
executeNonSourceStage(selfSchedulingInfo, rw, subscriptionStateHandlerRef);
}
}
logger.info("Calling lifecycle.shutdown()");
lifecycle.shutdown();
} catch (Throwable t) {
rw.signalFailed(t);
shutdownStage();
}
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceImplTest method testTimeoutShouldUnsubscribeServer.
@Test
public void testTimeoutShouldUnsubscribeServer() throws Exception {
HttpClientFactory<ByteBuf, ByteBuf> factory = new HttpClientFactory<ByteBuf, ByteBuf>() {
@Override
public HttpClient<ByteBuf, ByteBuf> createClient(ServerInfo server) {
ClientConfig clientConfig = new ClientConfig.Builder().readTimeout(10, TimeUnit.MILLISECONDS).build();
return new HttpClientBuilder<ByteBuf, ByteBuf>(server.getHost(), server.getPort()).config(clientConfig).build();
}
};
HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = HttpSourceImpl.builder(factory, HttpRequestFactories.createGetFactory("test/timeout?timeout=10000"), HttpSourceImpl.<ByteBuf>contextWrapper()).withActivityObserver(sourceObserver).resumeWith(new ClientResumePolicy<ByteBuf, ByteBuf>() {
@Override
public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
// TODO Auto-generated method stub
return null;
}
@Override
public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
// TODO Auto-generated method stub
return null;
}
}).withServerProvider(new HttpServerProvider() {
@Override
public Observable<ServerInfo> getServersToAdd() {
return Observable.from(localServerProvider.getServers()).map(new Func1<Server, ServerInfo>() {
@Override
public ServerInfo call(Server server) {
return toServerInfo(server);
}
});
}
@Override
public Observable<ServerInfo> getServersToRemove() {
return Observable.empty();
}
}).build();
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> ex = new AtomicReference<>();
final AtomicReference<ServerContext<ByteBuf>> items = new AtomicReference<>();
Observable.merge(source.call(new Context(), new Index(1, 1))).subscribe(new Subscriber<ServerContext<ByteBuf>>() {
@Override
public void onCompleted() {
latch.countDown();
}
@Override
public void onError(Throwable e) {
ex.set(e);
latch.countDown();
}
@Override
public void onNext(ServerContext<ByteBuf> pair) {
items.set(pair);
}
});
if (!latch.await(5, TimeUnit.SECONDS)) {
fail("The test case should finish way sooner than 5 seconds. ");
}
Assert.assertNull("The timeout error should be captured by the client so it does not surface to the source", ex.get());
Assert.assertNull("There should be no emitted item due to connection timeout", items.get());
for (Server server : localServerProvider.getServers()) {
ServerInfo serverInfo = toServerInfo(server);
assertEquals("There should be no source level error", 0, sourceObserver.getErrorCount());
assertEquals("There should be one connection attempt per server", 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ATTEMPTED));
assertEquals("There should be no established connection per server due to read timeout. ", 0, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ESTABLISHED));
assertEquals("There should no subscribed server because of read timeout", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
}
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceImplTest method testResumeOnCompletionButNotOnRemovedServers.
@Test
public void testResumeOnCompletionButNotOnRemovedServers() throws Exception {
final int maxRepeat = 10;
final int cutOff = 5;
final CountDownLatch countGate = new CountDownLatch(localServerProvider.serverSize() * (cutOff - 1));
final ConcurrentHashMap<ServerInfo, AtomicInteger> resumptionCounts = new ConcurrentHashMap<>();
final ConcurrentHashMap<ServerInfo, AtomicInteger> counterPerServer = new ConcurrentHashMap<>();
for (Server server : localServerProvider.getServers()) {
ServerInfo serverInfo = toServerInfo(server);
resumptionCounts.put(serverInfo, new AtomicInteger(0));
counterPerServer.put(serverInfo, new AtomicInteger(0));
}
HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = createSingleEntitySource(new ClientResumePolicy<ByteBuf, ByteBuf>() {
@Override
public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
return null;
}
@Override
public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
resumptionCounts.get(clientContext.getServer()).incrementAndGet();
countGate.countDown();
if (attempts < maxRepeat) {
return clientContext.newResponse();
} else {
return null;
}
}
});
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
Observable.merge(source.call(new Context(), new Index(1, 1))).map(new Func1<ServerContext<ByteBuf>, ServerContext<String>>() {
@Override
public ServerContext<String> call(ServerContext<ByteBuf> pair) {
return new ServerContext<>(pair.getServer(), pair.getValue().toString(Charset.defaultCharset()));
}
}).doOnNext(new Action1<ServerContext<String>>() {
@Override
public void call(ServerContext<String> context) {
counter.incrementAndGet();
assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, context.getValue());
ServerInfo server = context.getServer();
counterPerServer.get(server).incrementAndGet();
if (counterPerServer.get(server).get() > cutOff) {
localServerProvider.removeServer(server);
}
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
try {
countGate.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
done.countDown();
}
}).subscribe();
long wait = 5;
if (!done.await(wait, TimeUnit.SECONDS)) {
fail(String.format("All streaming should be done within %d seconds. ", wait));
}
assertEquals("All server should be resumed", localServerProvider.serverSize(), resumptionCounts.size());
for (ServerInfo server : resumptionCounts.keySet()) {
assertTrue(String.format("The server %s:%s should be resumed fewer than %d times", server.getHost(), server.getPort(), maxRepeat), maxRepeat > resumptionCounts.get(server).get());
}
assertTrue(String.format("There should be at least %d response from each of the %d servers", cutOff + 1, localServerProvider.serverSize()), (cutOff + 1) * localServerProvider.serverSize() <= counter.get());
for (Server server : localServerProvider.getServers()) {
ServerInfo serverInfo = toServerInfo(server);
assertEquals("There should be no error", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_FAILED));
}
assertEquals(String.format("There are %d repeats, but there should be only one final completion", maxRepeat - 1), 1, sourceObserver.getCompletionCount());
assertEquals("There should be no error", 0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
Set<EventType> expectedEvents = new HashSet<>();
expectedEvents.addAll(Arrays.asList(EventType.SUBSCRIPTION_CANCELED, EventType.SERVER_FOUND, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ENDED, EventType.CONNECTION_UNSUBSCRIBED));
assertEquals(expectedEvents, events);
assertEquals("Each connection should be unsubscribed once by the subscriber", localServerProvider.serverSize(), sourceObserver.getEventCount(CONNECTION_UNSUBSCRIBED));
for (EventType eventType : new EventType[] { EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED }) {
assertTrue(String.format("Event %s should be recorded at least %d times per server", eventType, cutOff), (cutOff + 1) * localServerProvider.serverSize() <= sourceObserver.getEventCount(eventType));
}
for (EventType eventType : new EventType[] { EventType.SERVER_FOUND, EventType.SUBSCRIPTION_CANCELED }) {
assertEquals(String.format("Event %s should be recorded exactly once per server", eventType), localServerProvider.serverSize(), sourceObserver.getEventCount(eventType));
}
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceImplTest method testResumeOnCompletion.
@Test
public void testResumeOnCompletion() throws Exception {
final int maxRepeat = 10;
final CountDownLatch countGate = new CountDownLatch(maxRepeat * localServerProvider.serverSize());
HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = createSingleEntitySource(new ClientResumePolicy<ByteBuf, ByteBuf>() {
@Override
public Observable<HttpClientResponse<ByteBuf>> onError(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts, Throwable error) {
return null;
}
@Override
public Observable<HttpClientResponse<ByteBuf>> onCompleted(ServerClientContext<ByteBuf, ByteBuf> clientContext, int attempts) {
countGate.countDown();
if (attempts < maxRepeat) {
return clientContext.newResponse();
} else {
return null;
}
}
});
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
Observable.merge(source.call(new Context(), new Index(1, 1))).map(new Func1<ServerContext<ByteBuf>, ServerContext<String>>() {
@Override
public ServerContext<String> call(ServerContext<ByteBuf> pair) {
try {
return new ServerContext<>(pair.getServer(), pair.getValue().retain().toString(Charset.defaultCharset()));
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}).doOnNext(new Action1<ServerContext<String>>() {
@Override
public void call(ServerContext<String> pair) {
counter.incrementAndGet();
assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, pair.getValue());
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
try {
countGate.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
done.countDown();
}
}).subscribe();
long wait = 5;
if (!done.await(5, TimeUnit.SECONDS)) {
fail(String.format("All streaming should be done within %d seconds. ", wait));
}
assertEquals(String.format("There should be exactly %d response from each of the %d servers", maxRepeat, localServerProvider.serverSize()), maxRepeat * localServerProvider.serverSize(), counter.get());
for (Server server : localServerProvider.getServers()) {
ServerInfo serverInfo = toServerInfo(server);
assertEquals(String.format("There should be %d completion per server as resumption function should called %d times", maxRepeat, maxRepeat - 1), maxRepeat, sourceObserver.getCount(serverInfo, EventType.SOURCE_COMPLETED));
assertEquals("There should be no error", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_FAILED));
assertEquals(String.format("Connection per server should have been established %d times", maxRepeat), 10, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
}
assertEquals(String.format("There are %d repeats, but there should be only one final completion", maxRepeat - 1), 1, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertEquals(EXPECTED_EVENTS_SETS, events);
assertEquals("Each connection should be unsubscribed once by the subscriber", localServerProvider.serverSize(), sourceObserver.getEventCount(CONNECTION_UNSUBSCRIBED));
for (EventType eventType : new EventType[] { EventType.SOURCE_COMPLETED, EventType.SUBSCRIPTION_ESTABLISHED, EventType.CONNECTION_ATTEMPTED, EventType.CONNECTION_ESTABLISHED }) {
assertEquals(String.format("Event %s should be recorded exactly %d times per server", eventType, maxRepeat), maxRepeat * localServerProvider.serverSize(), sourceObserver.getEventCount(eventType));
}
assertEquals("Event SERVER_FOUND should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(EventType.SERVER_FOUND));
}
use of io.mantisrx.runtime.Context in project mantis by Netflix.
the class HttpSourceTest method sourceEchoStreamFromPost.
private void sourceEchoStreamFromPost(HttpSource<ServerSentEvent, ServerSentEvent> source, String postContent) throws Exception {
final AtomicInteger counter = new AtomicInteger();
final CountDownLatch done = new CountDownLatch(1);
final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerSentEvent>() {
@Override
public void call(ServerSentEvent event) {
counter.incrementAndGet();
String msg = event.contentAsString();
result.putIfAbsent(msg, new AtomicInteger());
result.get(msg).incrementAndGet();
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable throwable) {
fail("Unexpected failure: " + throwable);
}
}).doAfterTerminate(new Action0() {
@Override
public void call() {
done.countDown();
}
}).subscribe();
long waitSeconds = 30000;
boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
if (timedout) {
fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
}
Assert.assertEquals(String.format("%d servers => the result has %d times of a single echo", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), counter.get());
assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(postContent).get());
for (ServerInfo server : localServerProvider.getServerInfos()) {
assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
}
assertEquals(1, sourceObserver.getCompletionCount());
assertEquals(0, sourceObserver.getErrorCount());
Set<EventType> events = sourceObserver.getEvents();
assertEquals(EXPECTED_EVENTS_SETS, events);
for (EventType event : events) {
assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
}
}
Aggregations