Search in sources :

Example 1 with GetJobSchedInfoRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoRequest in project mantis by Netflix.

the class JobDiscoveryRouteHandlerAkkaImpl method schedulingInfoStream.

@Override
public CompletionStage<JobDiscoveryRouteProto.SchedInfoResponse> schedulingInfoStream(final GetJobSchedInfoRequest request, final boolean sendHeartbeats) {
    CompletionStage<GetJobSchedInfoResponse> response = schedInfoCache.get(request);
    try {
        AtomicBoolean isJobCompleted = new AtomicBoolean(false);
        final String jobId = request.getJobId().getId();
        final JobSchedulingInfo completedJobSchedulingInfo = new JobSchedulingInfo(jobId, new HashMap<>());
        CompletionStage<JobDiscoveryRouteProto.SchedInfoResponse> jobSchedInfoObsCS = response.thenApply(getJobSchedInfoResp -> {
            Optional<BehaviorSubject<JobSchedulingInfo>> jobStatusSubjectO = getJobSchedInfoResp.getJobSchedInfoSubject();
            if (getJobSchedInfoResp.responseCode.equals(BaseResponse.ResponseCode.SUCCESS) && jobStatusSubjectO.isPresent()) {
                BehaviorSubject<JobSchedulingInfo> jobSchedulingInfoObs = jobStatusSubjectO.get();
                Observable<JobSchedulingInfo> heartbeats = Observable.interval(5, serverIdleConnectionTimeout.getSeconds() - 1, TimeUnit.SECONDS).map(x -> {
                    if (!isJobCompleted.get()) {
                        return SCHED_INFO_HB_INSTANCE;
                    } else {
                        return completedJobSchedulingInfo;
                    }
                }).takeWhile(x -> sendHeartbeats == true);
                // Job SchedulingInfo obs completes on job shutdown. Use the do On completed as a signal to inform the user that there are no workers to connect to.
                // TODO For future a more explicit key in the payload saying the job is completed.
                Observable<JobSchedulingInfo> jobSchedulingInfoWithHBObs = Observable.merge(jobSchedulingInfoObs.doOnCompleted(() -> isJobCompleted.set(true)), heartbeats);
                return new JobDiscoveryRouteProto.SchedInfoResponse(getJobSchedInfoResp.requestId, getJobSchedInfoResp.responseCode, getJobSchedInfoResp.message, jobSchedulingInfoWithHBObs);
            } else {
                logger.info("Failed to get Sched info stream for {}", request.getJobId().getId());
                schedInfoStreamErrors.increment();
                return new JobDiscoveryRouteProto.SchedInfoResponse(getJobSchedInfoResp.requestId, getJobSchedInfoResp.responseCode, getJobSchedInfoResp.message);
            }
        });
        return jobSchedInfoObsCS;
    } catch (Exception e) {
        logger.error("caught exception fetching sched info stream for {}", request.getJobId().getId(), e);
        schedInfoStreamErrors.increment();
        return CompletableFuture.completedFuture(new JobDiscoveryRouteProto.SchedInfoResponse(0, BaseResponse.ResponseCode.SERVER_ERROR, "Failed to get SchedulingInfo stream for jobId " + request.getJobId().getId() + " error: " + e.getMessage()));
    }
}
Also used : JobId(io.mantisrx.server.master.domain.JobId) JobClusterInfo(io.mantisrx.master.api.akka.route.proto.JobClusterInfo) GetJobSchedInfoRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoRequest) LoggerFactory(org.slf4j.LoggerFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) JOB_CLUSTER_INFO_HB_INSTANCE(io.mantisrx.master.api.akka.route.utils.JobDiscoveryHeartbeats.JOB_CLUSTER_INFO_HB_INSTANCE) JobDiscoveryRouteProto(io.mantisrx.master.api.akka.route.proto.JobDiscoveryRouteProto) GetLastSubmittedJobIdStreamResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamResponse) Observable(rx.Observable) ActorRef(akka.actor.ActorRef) Duration(java.time.Duration) Metrics(io.mantisrx.common.metrics.Metrics) GetLastSubmittedJobIdStreamRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamRequest) Counter(io.mantisrx.common.metrics.Counter) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) Caffeine(com.github.benmanes.caffeine.cache.Caffeine) SCHED_INFO_HB_INSTANCE(io.mantisrx.master.api.akka.route.utils.JobDiscoveryHeartbeats.SCHED_INFO_HB_INSTANCE) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) BaseResponse(io.mantisrx.master.jobcluster.proto.BaseResponse) GetJobSchedInfoResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoResponse) TimeUnit(java.util.concurrent.TimeUnit) AsyncLoadingCache(com.github.benmanes.caffeine.cache.AsyncLoadingCache) CompletionStage(java.util.concurrent.CompletionStage) PatternsCS.ask(akka.pattern.PatternsCS.ask) ConfigurationProvider(io.mantisrx.server.master.config.ConfigurationProvider) BehaviorSubject(rx.subjects.BehaviorSubject) Optional(java.util.Optional) BehaviorSubject(rx.subjects.BehaviorSubject) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) GetJobSchedInfoResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) GetJobSchedInfoResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoResponse)

Aggregations

ActorRef (akka.actor.ActorRef)1 PatternsCS.ask (akka.pattern.PatternsCS.ask)1 AsyncLoadingCache (com.github.benmanes.caffeine.cache.AsyncLoadingCache)1 Caffeine (com.github.benmanes.caffeine.cache.Caffeine)1 Counter (io.mantisrx.common.metrics.Counter)1 Metrics (io.mantisrx.common.metrics.Metrics)1 JobClusterInfo (io.mantisrx.master.api.akka.route.proto.JobClusterInfo)1 JobDiscoveryRouteProto (io.mantisrx.master.api.akka.route.proto.JobDiscoveryRouteProto)1 JOB_CLUSTER_INFO_HB_INSTANCE (io.mantisrx.master.api.akka.route.utils.JobDiscoveryHeartbeats.JOB_CLUSTER_INFO_HB_INSTANCE)1 SCHED_INFO_HB_INSTANCE (io.mantisrx.master.api.akka.route.utils.JobDiscoveryHeartbeats.SCHED_INFO_HB_INSTANCE)1 BaseResponse (io.mantisrx.master.jobcluster.proto.BaseResponse)1 GetJobSchedInfoRequest (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoRequest)1 GetJobSchedInfoResponse (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoResponse)1 GetLastSubmittedJobIdStreamRequest (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamRequest)1 GetLastSubmittedJobIdStreamResponse (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamResponse)1 JobSchedulingInfo (io.mantisrx.server.core.JobSchedulingInfo)1 ConfigurationProvider (io.mantisrx.server.master.config.ConfigurationProvider)1 JobId (io.mantisrx.server.master.domain.JobId)1 Duration (java.time.Duration)1 HashMap (java.util.HashMap)1