Examples with ToDeltaEndpointInjector io.reactivex.mantis.remote.observable.ToDeltaEndpointInjector used on opensource projects

Example 1 with ToDeltaEndpointInjector

use of io.reactivex.mantis.remote.observable.ToDeltaEndpointInjector in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method connectToObservableAtPreviousStages.

@SuppressWarnings({ "rawtypes" })
private WorkerConsumer connectToObservableAtPreviousStages(Observable<JobSchedulingInfo> selfSchedulingInfo, final String jobId, final int previousStageNum, int numInstanceAtPreviousStage, final StageConfig previousStage, final AtomicBoolean acceptSchedulingChanges, final Observer<Status> jobStatus, final int stageNumToExecute, final int workerIndex, final int workerNumber) {
    logger.info("Watching for scheduling changes");
    // Observable<List<Endpoint>> schedulingUpdates = mantisMasterApi.schedulingChanges(jobId)
    Observable<List<Endpoint>> schedulingUpdates = selfSchedulingInfo.flatMap((Func1<JobSchedulingInfo, Observable<WorkerAssignments>>) schedulingChange -> {
        Map<Integer, WorkerAssignments> assignments = schedulingChange.getWorkerAssignments();
        if (assignments != null && !assignments.isEmpty()) {
            return Observable.from(assignments.values());
        } else {
            return Observable.empty();
        }
    }).filter(assignments -> (assignments.getStage() == previousStageNum) && acceptSchedulingChanges.get()).map(assignments -> {
        List<Endpoint> endpoints = new LinkedList<>();
        for (WorkerHost host : assignments.getHosts().values()) {
            if (host.getState() == MantisJobState.Started) {
                logger.info("Received scheduling update from master, connect request for host: " + host.getHost() + " port: " + host.getPort() + " state: " + host.getState() + " adding: " + connectionsPerEndpoint + " connections to host");
                for (int i = 1; i <= connectionsPerEndpoint; i++) {
                    final String endpointId = "stage_" + stageNumToExecute + "_index_" + Integer.toString(workerIndex) + "_partition_" + i;
                    logger.info("Adding endpoint to endpoint injector to be considered for add, with id: " + endpointId);
                    endpoints.add(new Endpoint(host.getHost(), host.getPort().get(0), endpointId));
                }
            }
        }
        return endpoints;
    }).filter(t1 -> (t1.size() > 0));
    String name = jobId + "_" + previousStageNum;
    return new WorkerConsumerRemoteObservable(name, new ToDeltaEndpointInjector(schedulingUpdates));
}

Example 2 with ToDeltaEndpointInjector

use of io.reactivex.mantis.remote.observable.ToDeltaEndpointInjector in project mantis by Netflix.

the class MasterClientWrapper method getSinkLocations.

public Observable<EndpointChange> getSinkLocations(final String jobId, final int sinkStage, final int forPartition, final int totalPartitions) {
    final ConditionalRetry schedInfoRetry = new ConditionalRetry(masterConnectRetryCounter, "SchedInfoRetry", 10);
    Observable<List<Endpoint>> schedulingUpdates = getMasterClientApi().take(1).flatMap((MantisMasterClientApi mantisMasterClientApi) -> {
        return mantisMasterClientApi.schedulingChanges(jobId).doOnError((Throwable throwable) -> {
            logger.warn(throwable.getMessage());
        }).retryWhen(schedInfoRetry.getRetryLogic()).map((JobSchedulingInfo jobSchedulingInfo) -> {
            logger.info("Got scheduling info for {}", jobId);
            if (logger.isDebugEnabled()) {
                logger.debug("Worker Assignments {}", jobSchedulingInfo.getWorkerAssignments().get(sinkStage));
            }
            return jobSchedulingInfo.getWorkerAssignments().get(sinkStage);
        }).map((WorkerAssignments workerAssignments) -> {
            List<Endpoint> endpoints = new ArrayList<>();
            if (workerAssignments != null) {
                logger.info("job " + jobId + " Creating endpoints conx from " + workerAssignments.getHosts().size() + " worker assignments");
                for (WorkerHost host : workerAssignments.getHosts().values()) {
                    final int workerIndex = host.getWorkerIndex();
                    final int totalFromPartitions = workerAssignments.getNumWorkers();
                    numSinkWorkersSubject.onNext(new JobSinkNumWorkers(jobId, totalFromPartitions));
                    if (usePartition(workerIndex, totalFromPartitions, forPartition, totalPartitions)) {
                        // logger.info("Using partition " + workerIndex);
                        if (host.getState() == MantisJobState.Started) {
                            Endpoint ep = new Endpoint(getWrappedHost(host.getHost(), host.getWorkerNumber()), host.getPort().get(0), // completed callback
                            () -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " completed"), // error callback
                            t1 -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " failed"));
                            endpoints.add(ep);
                        }
                    }
                }
            } else {
                logger.info("job " + jobId + " Has no active workers!");
            }
            return endpoints;
        }).doOnError((Throwable throwable) -> {
            logger.error(throwable.getMessage(), throwable);
        });
    });
    return (new ToDeltaEndpointInjector(schedulingUpdates)).deltas();
}