use of io.mantisrx.runtime.executor.WorkerConsumer in project mantis by Netflix.
the class WorkerExecutionOperationsNetworkStage method connectToObservableAtPreviousStages.
@SuppressWarnings({ "rawtypes" })
private WorkerConsumer connectToObservableAtPreviousStages(Observable<JobSchedulingInfo> selfSchedulingInfo, final String jobId, final int previousStageNum, int numInstanceAtPreviousStage, final StageConfig previousStage, final AtomicBoolean acceptSchedulingChanges, final Observer<Status> jobStatus, final int stageNumToExecute, final int workerIndex, final int workerNumber) {
logger.info("Watching for scheduling changes");
// Observable<List<Endpoint>> schedulingUpdates = mantisMasterApi.schedulingChanges(jobId)
Observable<List<Endpoint>> schedulingUpdates = selfSchedulingInfo.flatMap((Func1<JobSchedulingInfo, Observable<WorkerAssignments>>) schedulingChange -> {
Map<Integer, WorkerAssignments> assignments = schedulingChange.getWorkerAssignments();
if (assignments != null && !assignments.isEmpty()) {
return Observable.from(assignments.values());
} else {
return Observable.empty();
}
}).filter(assignments -> (assignments.getStage() == previousStageNum) && acceptSchedulingChanges.get()).map(assignments -> {
List<Endpoint> endpoints = new LinkedList<>();
for (WorkerHost host : assignments.getHosts().values()) {
if (host.getState() == MantisJobState.Started) {
logger.info("Received scheduling update from master, connect request for host: " + host.getHost() + " port: " + host.getPort() + " state: " + host.getState() + " adding: " + connectionsPerEndpoint + " connections to host");
for (int i = 1; i <= connectionsPerEndpoint; i++) {
final String endpointId = "stage_" + stageNumToExecute + "_index_" + Integer.toString(workerIndex) + "_partition_" + i;
logger.info("Adding endpoint to endpoint injector to be considered for add, with id: " + endpointId);
endpoints.add(new Endpoint(host.getHost(), host.getPort().get(0), endpointId));
}
}
}
return endpoints;
}).filter(t1 -> (t1.size() > 0));
String name = jobId + "_" + previousStageNum;
return new WorkerConsumerRemoteObservable(name, new ToDeltaEndpointInjector(schedulingUpdates));
}
Aggregations