Search in sources :

Example 1 with WorkerConsumer

use of io.mantisrx.runtime.executor.WorkerConsumer in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method connectToObservableAtPreviousStages.

@SuppressWarnings({ "rawtypes" })
private WorkerConsumer connectToObservableAtPreviousStages(Observable<JobSchedulingInfo> selfSchedulingInfo, final String jobId, final int previousStageNum, int numInstanceAtPreviousStage, final StageConfig previousStage, final AtomicBoolean acceptSchedulingChanges, final Observer<Status> jobStatus, final int stageNumToExecute, final int workerIndex, final int workerNumber) {
    logger.info("Watching for scheduling changes");
    // Observable<List<Endpoint>> schedulingUpdates = mantisMasterApi.schedulingChanges(jobId)
    Observable<List<Endpoint>> schedulingUpdates = selfSchedulingInfo.flatMap((Func1<JobSchedulingInfo, Observable<WorkerAssignments>>) schedulingChange -> {
        Map<Integer, WorkerAssignments> assignments = schedulingChange.getWorkerAssignments();
        if (assignments != null && !assignments.isEmpty()) {
            return Observable.from(assignments.values());
        } else {
            return Observable.empty();
        }
    }).filter(assignments -> (assignments.getStage() == previousStageNum) && acceptSchedulingChanges.get()).map(assignments -> {
        List<Endpoint> endpoints = new LinkedList<>();
        for (WorkerHost host : assignments.getHosts().values()) {
            if (host.getState() == MantisJobState.Started) {
                logger.info("Received scheduling update from master, connect request for host: " + host.getHost() + " port: " + host.getPort() + " state: " + host.getState() + " adding: " + connectionsPerEndpoint + " connections to host");
                for (int i = 1; i <= connectionsPerEndpoint; i++) {
                    final String endpointId = "stage_" + stageNumToExecute + "_index_" + Integer.toString(workerIndex) + "_partition_" + i;
                    logger.info("Adding endpoint to endpoint injector to be considered for add, with id: " + endpointId);
                    endpoints.add(new Endpoint(host.getHost(), host.getPort().get(0), endpointId));
                }
            }
        }
        return endpoints;
    }).filter(t1 -> (t1.size() > 0));
    String name = jobId + "_" + previousStageNum;
    return new WorkerConsumerRemoteObservable(name, new ToDeltaEndpointInjector(schedulingUpdates));
}
Also used : Strings(io.mantisrx.shaded.com.google.common.base.Strings) Arrays(java.util.Arrays) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) LoggerFactory(org.slf4j.LoggerFactory) StageSchedulingInfo(io.mantisrx.runtime.descriptor.StageSchedulingInfo) JobMasterStageConfig(io.mantisrx.server.worker.jobmaster.JobMasterStageConfig) Lifecycle(io.mantisrx.runtime.lifecycle.Lifecycle) WorkerConsumer(io.mantisrx.runtime.executor.WorkerConsumer) ServiceRegistry(io.mantisrx.server.core.ServiceRegistry) JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM(io.mantisrx.runtime.parameter.ParameterUtils.JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM) WorkerPorts(io.mantisrx.common.WorkerPorts) ParameterUtils(io.mantisrx.runtime.parameter.ParameterUtils) Map(java.util.Map) Schedulers(rx.schedulers.Schedulers) VirtualMachineTaskStatus(io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus) RxMetrics(io.reactivex.mantis.remote.observable.RxMetrics) Status(io.mantisrx.server.core.Status) StageExecutors(io.mantisrx.runtime.executor.StageExecutors) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) Observer(rx.Observer) Collectors(java.util.stream.Collectors) JobMasterService(io.mantisrx.server.worker.jobmaster.JobMasterService) WorkerConsumerRemoteObservable(io.mantisrx.runtime.executor.WorkerConsumerRemoteObservable) CountDownLatch(java.util.concurrent.CountDownLatch) WorkerId(io.mantisrx.server.core.domain.WorkerId) List(java.util.List) ToDeltaEndpointInjector(io.reactivex.mantis.remote.observable.ToDeltaEndpointInjector) Action0(rx.functions.Action0) BehaviorSubject(rx.subjects.BehaviorSubject) Splitter(io.mantisrx.shaded.com.google.common.base.Splitter) Optional(java.util.Optional) WorkerMap(io.mantisrx.runtime.WorkerMap) PortSelector(io.mantisrx.runtime.executor.PortSelector) WorkerPublisherRemoteObservable(io.mantisrx.runtime.executor.WorkerPublisherRemoteObservable) StageConfig(io.mantisrx.runtime.StageConfig) MantisMasterClientApi(io.mantisrx.server.master.client.MantisMasterClientApi) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Parameters(io.mantisrx.runtime.parameter.Parameters) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Observable(rx.Observable) Func1(rx.functions.Func1) WorkerMetricsClient(io.mantisrx.server.worker.client.WorkerMetricsClient) LinkedList(java.util.LinkedList) RemoteRxServer(io.reactivex.mantis.remote.observable.RemoteRxServer) AutoScaleMetricsConfig(io.mantisrx.server.worker.jobmaster.AutoScaleMetricsConfig) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Endpoint(io.mantisrx.common.network.Endpoint) TYPE(io.mantisrx.server.core.Status.TYPE) Context(io.mantisrx.runtime.Context) StatusPayloads(io.mantisrx.server.core.StatusPayloads) TimeUnit(java.util.concurrent.TimeUnit) ServiceLocator(io.mantisrx.runtime.lifecycle.ServiceLocator) ExecuteStageRequest(io.mantisrx.server.core.ExecuteStageRequest) Registry(com.netflix.spectator.api.Registry) WorkerConfiguration(io.mantisrx.server.worker.config.WorkerConfiguration) SpectatorRegistryFactory(io.mantisrx.common.metrics.spectator.SpectatorRegistryFactory) WorkerInfo(io.mantisrx.runtime.WorkerInfo) WorkerHost(io.mantisrx.server.core.WorkerHost) WorkerHost(io.mantisrx.server.core.WorkerHost) ToDeltaEndpointInjector(io.reactivex.mantis.remote.observable.ToDeltaEndpointInjector) Endpoint(io.mantisrx.common.network.Endpoint) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) WorkerConsumerRemoteObservable(io.mantisrx.runtime.executor.WorkerConsumerRemoteObservable) List(java.util.List) LinkedList(java.util.LinkedList) Func1(rx.functions.Func1)

Aggregations

Registry (com.netflix.spectator.api.Registry)1 WorkerPorts (io.mantisrx.common.WorkerPorts)1 MetricsRegistry (io.mantisrx.common.metrics.MetricsRegistry)1 SpectatorRegistryFactory (io.mantisrx.common.metrics.spectator.SpectatorRegistryFactory)1 Endpoint (io.mantisrx.common.network.Endpoint)1 Context (io.mantisrx.runtime.Context)1 MantisJobDurationType (io.mantisrx.runtime.MantisJobDurationType)1 MantisJobState (io.mantisrx.runtime.MantisJobState)1 StageConfig (io.mantisrx.runtime.StageConfig)1 WorkerInfo (io.mantisrx.runtime.WorkerInfo)1 WorkerMap (io.mantisrx.runtime.WorkerMap)1 StageSchedulingInfo (io.mantisrx.runtime.descriptor.StageSchedulingInfo)1 PortSelector (io.mantisrx.runtime.executor.PortSelector)1 StageExecutors (io.mantisrx.runtime.executor.StageExecutors)1 WorkerConsumer (io.mantisrx.runtime.executor.WorkerConsumer)1 WorkerConsumerRemoteObservable (io.mantisrx.runtime.executor.WorkerConsumerRemoteObservable)1 WorkerPublisherRemoteObservable (io.mantisrx.runtime.executor.WorkerPublisherRemoteObservable)1 Lifecycle (io.mantisrx.runtime.lifecycle.Lifecycle)1 ServiceLocator (io.mantisrx.runtime.lifecycle.ServiceLocator)1 ParameterUtils (io.mantisrx.runtime.parameter.ParameterUtils)1