Examples with JobClient com.dtstack.taier.pluginapi.JobClient used on opensource projects

Example 21 with JobClient

use of com.dtstack.taier.pluginapi.JobClient in project Taier by DTStack.

the class JobDealer method addSubmitJobVast.

/**
 * job cache 表已经存在
 * @param jobClients
 */
private void addSubmitJobVast(List<JobClient> jobClients) {
    List<String> jobIds = jobClients.stream().map(JobClient::getJobId).collect(Collectors.toList());
    updateCacheBatch(jobIds, EJobCacheStage.DB.getStage());
    scheduleJobService.updateJobStatusByJobIds(jobIds, TaskStatus.WAITENGINE.getStatus(), null);
    LOGGER.info(" addSubmitJobBatch jobId:{} update", JSONObject.toJSONString(jobIds));
    for (JobClient jobClient : jobClients) {
        jobClient.setCallBack((jobStatus) -> {
            updateJobStatus(jobClient.getJobId(), jobStatus);
        });
        // 加入节点的优先级队列
        this.addGroupPriorityQueue(jobComputeResourcePlain.getJobResource(jobClient), jobClient, true, false);
    }
}

Example 22 with JobClient

use of com.dtstack.taier.pluginapi.JobClient in project Taier by DTStack.

the class JobStopDealer method stopJob.

private StoppedStatus stopJob(JobElement jobElement) throws Exception {
    ScheduleEngineJobCache jobCache = engineJobCacheService.getByJobId(jobElement.jobId);
    ScheduleJob scheduleJob = scheduleJobService.lambdaQuery().eq(ScheduleJob::getJobId, jobElement.jobId).eq(ScheduleJob::getIsDeleted, Deleted.NORMAL.getStatus()).one();
    if (jobCache == null) {
        if (scheduleJob != null && TaskStatus.isStopped(scheduleJob.getStatus())) {
            LOGGER.info("jobId:{} stopped success, set job is STOPPED.", jobElement.jobId);
            return StoppedStatus.STOPPED;
        } else {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} jobCache is null, set job is MISSED.", jobElement.jobId);
            return StoppedStatus.MISSED;
        }
    } else if (null != scheduleJob && EJobCacheStage.unSubmitted().contains(jobCache.getStage())) {
        if (!TaskStatus.getWaitStatus().contains(scheduleJob.getStatus()) || EJobCacheStage.PRIORITY.getStage() != jobCache.getStage()) {
            this.removeMemStatusAndJobCache(jobCache.getJobId());
            LOGGER.info("jobId:{} is unsubmitted, set job is STOPPED.", jobElement.jobId);
            return StoppedStatus.STOPPED;
        } else {
            // 任务如果处于提交的状态过程中 但是stage由PRIORITY变更为SUBMITTED  直接删除会导致还是会提交到yarn上 占用资源
            LOGGER.info("jobId:{} is stopping.", jobCache.getJobId());
            return StoppedStatus.STOPPING;
        }
    } else {
        if (scheduleJob == null) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} scheduleJob is null, set job is MISSED.", jobElement.jobId);
            return StoppedStatus.MISSED;
        } else if (TaskStatus.getStoppedAndNotFound().contains(scheduleJob.getStatus())) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} and status:{} is StoppedAndNotFound, set job is STOPPED.", jobElement.jobId, scheduleJob.getStatus());
            return StoppedStatus.STOPPED;
        }
        ParamAction paramAction = PublicUtil.jsonStrToObject(jobCache.getJobInfo(), ParamAction.class);
        paramAction.setEngineTaskId(scheduleJob.getEngineJobId());
        paramAction.setApplicationId(scheduleJob.getApplicationId());
        JobClient jobClient = new JobClient(paramAction);
        jobClient.setForceCancel(jobElement.isForceCancel);
        if (StringUtils.isNotBlank(scheduleJob.getEngineJobId()) && !jobClient.getEngineTaskId().equals(scheduleJob.getEngineJobId())) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} stopped success, because of [difference engineJobId].", paramAction.getJobId());
            return StoppedStatus.STOPPED;
        }
        JobResult jobResult = workerOperator.stopJob(jobClient);
        if (jobResult.getCheckRetry()) {
            LOGGER.info("jobId:{} is retry.", paramAction.getJobId());
            return StoppedStatus.RETRY;
        } else {
            LOGGER.info("jobId:{} is stopping.", paramAction.getJobId());
            return StoppedStatus.STOPPING;
        }
    }
}

Example 23 with JobClient

use of com.dtstack.taier.pluginapi.JobClient in project Taier by DTStack.

the class JobSubmitDealer method run.

@Override
public void run() {
    while (true) {
        try {
            JobClient jobClient = queue.take();
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("jobId:{} jobResource:{} queue size:{} take job from priorityQueue.", jobClient.getJobId(), jobResource, queue.size());
            }
            if (checkIsFinished(jobClient)) {
                continue;
            }
            if (checkJobSubmitExpired(jobClient)) {
                shardCache.updateLocalMemTaskStatus(jobClient.getJobId(), TaskStatus.AUTOCANCELED.getStatus());
                jobClient.doStatusCallBack(TaskStatus.AUTOCANCELED.getStatus());
                engineJobCacheService.deleteByJobId(jobClient.getJobId());
                LOGGER.info("jobId:{} checkJobSubmitExpired is true, job ignore to submit.", jobClient.getJobId());
                continue;
            }
            if (!checkMaxPriority(jobResource)) {
                LOGGER.info("jobId:{} checkMaxPriority is false, wait other node job which priority higher.", jobClient.getJobId());
                queue.put(jobClient);
                Thread.sleep(jobLackingInterval);
                continue;
            }
            // 提交任务
            jobSubmitConcurrentService.submit(() -> {
                submitJob(jobClient);
            });
        } catch (Exception e) {
            LOGGER.error("", e);
        }
    }
}

Example 24 with JobClient

use of com.dtstack.taier.pluginapi.JobClient in project Taier by DTStack.

the class JobSubmittedDealer method run.

@Override
public void run() {
    while (true) {
        JobClient jobClient = null;
        try {
            jobClient = queue.take();
            if (jobRestartDealer.checkAndRestartForSubmitResult(jobClient)) {
                LOGGER.warn("failed submit job restarting, jobId:{} jobResult:{} ...", jobClient.getJobId(), jobClient.getJobResult());
                continue;
            }
            LOGGER.info("success submit job to Engine, jobId:{} jobResult:{} ...", jobClient.getJobId(), jobClient.getJobResult());
            // 存储执行日志
            if (StringUtils.isNotBlank(jobClient.getEngineTaskId()) || StringUtils.isNotBlank(jobClient.getApplicationId())) {
                JobResult jobResult = jobClient.getJobResult();
                String appId = jobResult.getData(JobResult.JOB_ID_KEY);
                JSONObject jobExtraInfo = jobResult.getExtraInfoJson();
                jobExtraInfo.put(JobResultConstant.JOB_GRAPH, JobGraphUtil.formatJSON(appId, jobExtraInfo.getString(JobResultConstant.JOB_GRAPH), jobClient.getComputeType()));
                scheduleJobService.updateJobSubmitSuccess(jobClient.getJobId(), jobClient.getEngineTaskId(), appId);
                jobDealer.updateCache(jobClient, EJobCacheStage.SUBMITTED.getStage());
                jobClient.doStatusCallBack(TaskStatus.SUBMITTED.getStatus());
                JobClient finalJobClient = jobClient;
                shardCache.updateLocalMemTaskStatus(jobClient.getJobId(), TaskStatus.SUBMITTED.getStatus(), (jobId) -> {
                    LOGGER.warn("success submit job to Engine, jobId:{} jobResult:{} but shareManager is not found ...", jobId, finalJobClient.getJobResult());
                    finalJobClient.doStatusCallBack(TaskStatus.CANCELED.getStatus());
                });
            } else {
                jobClientFail(jobClient.getJobId(), jobClient.getJobResult().getJsonStr());
            }
        } catch (Throwable e) {
            LOGGER.error("jobId submitted {} jobStatus dealer run error", null == jobClient ? "" : jobClient.getJobId(), e);
            if (null != jobClient) {
                jobClientFail(jobClient.getJobId(), JobResult.createErrorResult(e).getJsonStr());
            }
        }
    }
}

Example 25 with JobClient

use of com.dtstack.taier.pluginapi.JobClient in project Taier by DTStack.

the class LogAspect method afterReturningAdvice.

@AfterReturning(pointcut = "execution(public * com.dtstack.taier.scheduler.WorkerOperator.*(..))", returning = "ret")
public void afterReturningAdvice(JoinPoint joinPoint, Object ret) {
    try {
        String methodName = joinPoint.getSignature().getName();
        if (filterMethod.contains(methodName)) {
            return;
        }
        String argsString = null;
        Object[] args = joinPoint.getArgs();
        Optional<Object> jobClientOpt = Arrays.stream(args).filter(a -> a instanceof JobClient).findFirst();
        if (jobClientOpt.isPresent()) {
            if (logPluginInfoMethod.contains(methodName)) {
                argsString = JSONObject.toJSONString(jobClientOpt.get(), submitPropertyFilter);
            } else {
                // 忽略pluginInfo打印
                argsString = JSONObject.toJSONString(jobClientOpt.get(), propertyFilter);
            }
        } else {
            if (skipChangeMethod.contains(methodName)) {
                if (ret instanceof TaskStatus && (TaskStatus.RUNNING.equals(ret) || TaskStatus.SCHEDULED.equals(ret))) {
                    // 状态获取 多以running 为主 过滤频繁打印
                    return;
                }
            } else {
                argsString = JSONObject.toJSONString(args);
            }
        }
        if (LOGGER.isInfoEnabled()) {
            JSONObject logInfo = new JSONObject(3);
            logInfo.put("method", joinPoint.getSignature().getDeclaringTypeName() + "." + methodName);
            logInfo.put("args", argsString);
            logInfo.put("return", JSONObject.toJSONString(ret));
            LOGGER.info(logInfo.toJSONString());
        }
    } catch (Exception e) {
        LOGGER.error("logAspect error ", e);
    }
}