Examples with CompletedJob io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob used on opensource projects

Example 1 with CompletedJob

use of io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob in project mantis by Netflix.

the class SimpleCachedFileStorageProvider method loadAllCompletedJobs.

@Override
public List<CompletedJob> loadAllCompletedJobs() throws IOException {
    createDir(JOB_CLUSTERS_DIR);
    List<CompletedJob> completedJobs = Lists.newArrayList();
    File clustersDir = new File(JOB_CLUSTERS_DIR);
    for (File jobClusterFile : clustersDir.listFiles((dir, name) -> name.endsWith(JOB_CLUSTERS_COMPLETED_JOBS_FILE_NAME_SUFFIX))) {
        try (FileInputStream fis = new FileInputStream(jobClusterFile)) {
            final List<CompletedJob> list = mapper.readValue(fis, new TypeReference<List<CompletedJob>>() {
            });
            if (list != null && !list.isEmpty())
                list.forEach(completedJobs::add);
        } catch (Exception e) {
            logger.error("Exception loading completedJob ", e);
        }
    }
    return completedJobs;
}

Example 2 with CompletedJob

use of io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob in project mantis by Netflix.

the class JobClusterActor method getFilteredTerminalJobList.

/**
 * JobState ActiveOnly Execute?
 *   None    None        Y
 *   None    TRUE        N
 *   None    FALSE       Y
 *   Active  None        N
 *   Active  TRUE        N
 *   Active  FALSE       N
 *   Terminal None       Y
 *   Terminal TRUE       Y
 *   Terminal FALSE      Y
 * @param request
 * @return
 */
private Observable<MantisJobMetadataView> getFilteredTerminalJobList(ListJobCriteria request, Set<JobId> jobIdSet) {
    if (logger.isTraceEnabled()) {
        logger.trace("JobClusterActor:getFilteredTerminalJobList");
    }
    if ((request.getJobState().isPresent() && !request.getJobState().get().equals(JobState.MetaState.Terminal))) {
        if (logger.isTraceEnabled()) {
            logger.trace("Exit JobClusterActor:getFilteredTerminalJobList with empty");
        }
        return Observable.empty();
    } else if (!request.getJobState().isPresent() && (request.getActiveOnly().isPresent() && request.getActiveOnly().get())) {
        if (logger.isTraceEnabled()) {
            logger.trace("Exit JobClusterActor:getFilteredTerminalJobList with empty");
        }
        return Observable.empty();
    }
    List<CompletedJob> jobInfoList;
    if (!jobIdSet.isEmpty()) {
        jobInfoList = jobIdSet.stream().map((jId) -> jobManager.getCompletedJob(jId)).filter((compJobOp) -> compJobOp.isPresent()).map((compJobOp) -> compJobOp.get()).collect(Collectors.toList());
    } else {
        jobInfoList = jobManager.getCompletedJobsList();
    }
    List<CompletedJob> shortenedList = jobInfoList.subList(0, Math.min(jobInfoList.size(), request.getLimit().orElse(DEFAULT_LIMIT)));
    return Observable.from(shortenedList).flatMap((cJob) -> {
        try {
            if (logger.isDebugEnabled()) {
                logger.debug("Fetching details for completed job {}", cJob);
            }
            Optional<IMantisJobMetadata> metaOp = jobManager.getJobDataForCompletedJob(cJob.getJobId());
            if (metaOp.isPresent()) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Fetched details for completed job {} -> {}", cJob, metaOp.get());
                }
                return Observable.just(new MantisJobMetadataView(metaOp.get(), cJob.getTerminatedAt(), request.getStageNumberList(), request.getWorkerIndexList(), request.getWorkerNumberList(), request.getWorkerStateList(), false));
            }
        } catch (Exception e) {
            logger.error("caught exception", e);
            return Observable.empty();
        }
        return Observable.empty();
    });
}

Example 3 with CompletedJob

use of io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob in project mantis by Netflix.

the class JobClusterActor method onGetJobDetailsRequest.

@Override
public void onGetJobDetailsRequest(GetJobDetailsRequest req) {
    if (logger.isTraceEnabled()) {
        logger.trace("Enter GetJobDetails {}", req);
    }
    GetJobDetailsResponse response = new GetJobDetailsResponse(req.requestId, CLIENT_ERROR_NOT_FOUND, "Job " + req.getJobId() + "  not found", empty());
    Optional<JobInfo> jInfo = jobManager.getJobInfoForNonTerminalJob(req.getJobId());
    if (jInfo.isPresent()) {
        if (logger.isDebugEnabled()) {
            logger.debug("Forwarding getJobDetails to job actor for {}", req.getJobId());
        }
        jInfo.get().jobActor.forward(req, getContext());
        return;
    } else {
        // Could be a terminated job
        Optional<CompletedJob> completedJob = jobManager.getCompletedJob(req.getJobId());
        if (completedJob.isPresent()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Found Job {} in completed state ", req.getJobId());
            }
            try {
                Optional<IMantisJobMetadata> jobMetaOp = jobStore.getArchivedJob(req.getJobId().getId());
                if (jobMetaOp.isPresent()) {
                    response = new GetJobDetailsResponse(req.requestId, SUCCESS, "", jobMetaOp);
                } else {
                    response = new GetJobDetailsResponse(req.requestId, CLIENT_ERROR_NOT_FOUND, "Job " + req.getJobId() + "  not found", empty());
                }
            } catch (Exception e) {
                logger.warn("Exception {} reading Job {} from Storage ", e.getMessage(), req.getJobId());
                response = new GetJobDetailsResponse(req.requestId, CLIENT_ERROR, "Exception reading Job " + req.getJobId() + "  " + e.getMessage(), empty());
            }
        } else {
            logger.warn("No such job {} ", req.getJobId());
        }
    }
    getSender().tell(response, getSelf());
    if (logger.isTraceEnabled()) {
        logger.trace("Exit GetJobDetails {}", req);
    }
}

Example 4 with CompletedJob

use of io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob in project mantis by Netflix.

the class JobClusterActor method onJobListCompleted.

@Override
public void onJobListCompleted(final ListCompletedJobsInClusterRequest request) {
    if (logger.isTraceEnabled()) {
        logger.trace("Enter onJobListCompleted {}", request);
    }
    final ActorRef sender = getSender();
    List<CompletedJob> completedJobsList = jobManager.getCompletedJobsList();
    if (request.getLimit() > completedJobsList.size()) {
        completedJobsList = completedJobsList.subList(0, request.getLimit());
    }
    sender.tell(new ListCompletedJobsInClusterResponse(request.requestId, SUCCESS, "", completedJobsList), sender);
    if (logger.isTraceEnabled()) {
        logger.trace("Exit onJobListCompleted {}", completedJobsList.size());
    }
}

Example 5 with CompletedJob

use of io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob in project mantis by Netflix.

the class JobClusterActor method initRunningJobs.

/**
 * Iterate through list of jobs in Active jobs table.
 * if a Job is completed move it completed table
 * else bootstrap the job (create actor, send init request)
 * Finally setup sla enforcement
 * @param initReq
 * @param sender
 */
private void initRunningJobs(JobClusterProto.InitializeJobClusterRequest initReq, ActorRef sender) {
    List<CompletedJob> completedJobsList = initReq.completedJobsList;
    List<IMantisJobMetadata> jobList = initReq.jobList;
    logger.info("In _initJobs for cluster {}: {} activeJobs and {} completedJobs", name, jobList.size(), completedJobsList.size());
    if (logger.isDebugEnabled()) {
        logger.debug("In _initJobs for cluster {} activeJobs -> {} and completedJobs -> {}", name, jobList, completedJobsList);
    }
    Observable.from(jobList).flatMap((jobMeta) -> {
        if (JobState.isTerminalState(jobMeta.getState())) {
            jobManager.persistToCompletedJobAndArchiveJobTables(jobMeta);
            return Observable.empty();
        } else {
            if (jobMeta.getSchedulingInfo() == null) {
                logger.error("Scheduling info is null for active job {} in cluster {}." + "Skipping bootstrap ", jobMeta.getJobId(), name);
                return Observable.empty();
            } else {
                return Observable.just(jobMeta);
            }
        }
    }).flatMap((jobMeta) -> jobManager.bootstrapJob((MantisJobMetadataImpl) jobMeta, this.jobClusterMetadata)).subscribe((jobInited) -> {
        logger.info("Job Id {} initialized with code {}", jobInited.jobId, jobInited.responseCode);
    }, (error) -> logger.warn("Exception initializing jobs {}", error.getMessage()), () -> {
        if (initReq.jobList.size() > 0) {
            JobId lastJobId = new JobId(this.name, initReq.lastJobNumber);
            this.jobIdSubmissionSubject.onNext(lastJobId);
        }
        setBookkeepingTimer(BOOKKEEPING_INTERVAL_SECS);
        getContext().become(initializedBehavior);
        logger.info("Job Cluster {} initialized", this.name);
        sender.tell(new JobClusterProto.InitializeJobClusterResponse(initReq.requestId, SUCCESS, String.format("JobCluster %s initialized successfully", initReq.jobClusterDefinition.getName()), initReq.jobClusterDefinition.getName(), initReq.requestor), getSelf());
    });
}