Examples with ScheduleJobDetails com.dtstack.taier.scheduler.server.ScheduleJobDetails used on opensource projects

Example 1 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class CycleJobBuilder method buildTaskJobGraph.

public void buildTaskJobGraph(String triggerDay) {
    if (environmentContext.getJobGraphBuilderSwitch()) {
        return;
    }
    lock.lock();
    try {
        String triggerTimeStr = triggerDay + " 00:00:00";
        Timestamp triggerTime = Timestamp.valueOf(triggerTimeStr);
        boolean hasBuild = jobGraphTriggerService.checkHasBuildJobGraph(triggerTime);
        if (hasBuild) {
            LOGGER.info("trigger Day {} has build so break", triggerDay);
            return;
        }
        // 1. 获得今天预计要生成的所有周期实例
        Integer totalTask = getTotalTask();
        LOGGER.info("{} need build job : {}", triggerTimeStr, totalTask);
        if (totalTask <= 0) {
            saveJobGraph(triggerDay);
            return;
        }
        clearInterruptJob(triggerTime);
        // 2. 切割总数 限制 thread 并发
        int totalBatch = totalTask / environmentContext.getJobLimitSize();
        if (totalTask % environmentContext.getJobLimitSize() != 0) {
            totalBatch++;
        }
        Semaphore sph = new Semaphore(environmentContext.getMaxTaskBuildThread());
        CountDownLatch ctl = new CountDownLatch(totalBatch);
        AtomicJobSortWorker sortWorker = new AtomicJobSortWorker();
        // 3. 查询db多线程生成周期实例
        Long startId = 0L;
        for (int i = 0; i < totalBatch; i++) {
            // 取50个任务
            final List<ScheduleTaskShade> batchTaskShades = scheduleTaskService.listRunnableTask(startId, Lists.newArrayList(EScheduleStatus.NORMAL.getVal(), EScheduleStatus.FREEZE.getVal()), environmentContext.getJobLimitSize());
            // 如果取出来的任务集合是空的
            if (CollectionUtils.isEmpty(batchTaskShades)) {
                continue;
            }
            startId = batchTaskShades.get(batchTaskShades.size() - 1).getId();
            LOGGER.info("job-number:{} startId:{}", i, startId);
            try {
                sph.acquire();
                jobGraphBuildPool.submit(() -> {
                    try {
                        for (ScheduleTaskShade batchTaskShade : batchTaskShades) {
                            try {
                                List<ScheduleJobDetails> scheduleJobDetails = RetryUtil.executeWithRetry(() -> buildJob(batchTaskShade, triggerDay, sortWorker), environmentContext.getBuildJobErrorRetry(), 200, false);
                                // 插入周期实例
                                savaJobList(scheduleJobDetails);
                            } catch (Throwable e) {
                                LOGGER.error("build task failure taskId:{} apptype:{}", batchTaskShade.getTaskId(), null, e);
                            }
                        }
                    } catch (Throwable e) {
                        LOGGER.error("!!! buildTaskJobGraph  build job error !!!", e);
                    } finally {
                        sph.release();
                        ctl.countDown();
                    }
                });
            } catch (Throwable e) {
                LOGGER.error("[acquire pool error]:", e);
                throw new RdosDefineException(e);
            }
        }
        ctl.await();
        // 循环已经结束，说明周期实例已经全部生成了
        saveJobGraph(triggerDay);
    } catch (Exception e) {
        LOGGER.error("buildTaskJobGraph ！！！", e);
    } finally {
        LOGGER.info("buildTaskJobGraph exit & unlock ...");
        lock.unlock();
    }
}

Example 2 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class FillDataJobBuilder method savaFillJob.

/**
 * 持久化时间
 *
 * @param allJobList 所有集合
 */
private void savaFillJob(List<ScheduleJobDetails> allJobList) {
    scheduleJobService.insertJobList(allJobList, EScheduleType.FILL_DATA.getType());
    List<ScheduleJobOperatorRecord> operatorJobIds = allJobList.stream().map(jobBuilderBean -> {
        ScheduleJobOperatorRecord record = new ScheduleJobOperatorRecord();
        record.setJobId(jobBuilderBean.getScheduleJob().getJobId());
        record.setForceCancelFlag(ForceCancelFlag.NO.getFlag());
        record.setOperatorType(OperatorType.FILL_DATA.getType());
        record.setNodeAddress(jobBuilderBean.getScheduleJob().getNodeAddress());
        return record;
    }).collect(Collectors.toList());
    scheduleJobOperatorRecordService.saveBatch(operatorJobIds);
}

Example 3 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class AbstractJobScanningScheduler method scanningJob.

/**
 * 扫描实例
 */
private void scanningJob() {
    try {
        String nodeAddress = zkService.getLocalAddress();
        // 1. 获得节点信息
        if (StringUtils.isBlank(nodeAddress)) {
            return;
        }
        LOGGER.info("scanningJob start scheduleType : {} nodeAddress:{}", getSchedulerName(), nodeAddress);
        // 2. 获得排序最小序号
        Long minSort = getMinSort();
        LOGGER.info("scanning start param: scheduleType {} nodeAddress {} minSort {} ", getSchedulerName(), nodeAddress, minSort);
        // 3. 扫描实例
        List<ScheduleJobDetails> scheduleJobDetails = listExecJob(minSort, nodeAddress, Boolean.TRUE);
        while (CollectionUtils.isNotEmpty(scheduleJobDetails)) {
            // 查询任务
            List<Long> taskIds = scheduleJobDetails.stream().map(ScheduleJobDetails::getScheduleJob).map(ScheduleJob::getTaskId).collect(Collectors.toList());
            Map<Long, ScheduleTaskShade> scheduleTaskShadeMap = scheduleTaskService.lambdaQuery().in(ScheduleTaskShade::getTaskId, taskIds).eq(ScheduleTaskShade::getIsDeleted, Deleted.NORMAL.getStatus()).list().stream().collect(Collectors.toMap(ScheduleTaskShade::getTaskId, g -> (g)));
            for (ScheduleJobDetails scheduleJobDetail : scheduleJobDetails) {
                // 提交实例
                ScheduleJob scheduleJob = scheduleJobDetail.getScheduleJob();
                ScheduleTaskShade scheduleTaskShade = scheduleTaskShadeMap.get(scheduleJob.getTaskId());
                if (scheduleTaskShade == null) {
                    String errMsg = JobCheckStatus.NO_TASK.getMsg();
                    scheduleJobService.updateStatusAndLogInfoById(scheduleJob.getJobId(), TaskStatus.SUBMITFAILD.getStatus(), errMsg);
                    LOGGER.warn("jobId:{} scheduleType:{} submit failed for taskId:{} already deleted.", scheduleJob.getJobId(), getSchedulerName(), scheduleJob.getTaskId());
                    continue;
                }
                scheduleJobDetail.setScheduleTaskShade(scheduleTaskShade);
                if (isSubmitJob(scheduleJobDetail)) {
                    submitJob(scheduleJobDetail);
                }
                if (minSort < scheduleJob.getJobExecuteOrder()) {
                    minSort = scheduleJob.getJobExecuteOrder();
                }
            }
            scheduleJobDetails = listExecJob(minSort, nodeAddress, Boolean.FALSE);
        }
    } catch (Exception e) {
        LOGGER.error("scheduleType:{} emitJob2Queue error:", getSchedulerName(), e);
    }
}

Example 4 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class AbstractJobSummitScheduler method run.

@Override
public void run() {
    while (RUNNING.get()) {
        ScheduleJob scheduleJob = null;
        try {
            ScheduleJobDetails scheduleJobDetails = scheduleJobQueue.take();
            scheduleJob = scheduleJobDetails.getScheduleJob();
            LOGGER.info("jobId:{} scheduleType:{} take job from queue.", scheduleJob.getJobId(), getSchedulerName());
            this.submit(scheduleJobDetails);
        } catch (InterruptedException ie) {
        // swallow the interrupt as it's only possible from either a background
        // operation and, thus, doesn't apply to this loop or the instance
        // is being closed in which case the while test will get it
        } catch (Exception e) {
            LOGGER.error("happens error:", e);
            try {
                if (scheduleJob != null) {
                    scheduleJobService.updateStatusAndLogInfoById(scheduleJob.getJobId(), TaskStatus.SUBMITFAILD.getStatus(), e.getMessage());
                    LOGGER.error("jobId:{} scheduleType:{} submit failed.", scheduleJob.getJobId(), getSchedulerName());
                }
            } catch (Exception ex) {
                LOGGER.error("scheduleType:{} update status happens error:", getSchedulerName(), ex);
            }
        }
    }
}

Example 5 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class JobUpStreamJudgeJobExecOperator method isExec.

@Override
public JobCheckRunInfo isExec(ScheduleJobDetails scheduleJobDetails) {
    List<ScheduleJobJob> jobJobList = scheduleJobDetails.getJobJobList();
    ScheduleJob scheduleJob = scheduleJobDetails.getScheduleJob();
    JobCheckRunInfo checkRunInfo = new JobCheckRunInfo();
    if (CollectionUtils.isNotEmpty(jobJobList)) {
        List<String> parentJobKeys = jobJobList.stream().map(ScheduleJobJob::getParentJobKey).collect(Collectors.toList());
        Map<String, ScheduleJob> scheduleJobMap = scheduleJobService.lambdaQuery().select(ScheduleJob::getStatus, ScheduleJob::getJobId, ScheduleJob::getJobKey, ScheduleJob::getJobName).in(ScheduleJob::getJobKey, parentJobKeys).eq(ScheduleJob::getIsDeleted, Deleted.NORMAL.getStatus()).list().stream().collect(Collectors.toMap(ScheduleJob::getJobKey, g -> (g)));
        for (ScheduleJobJob scheduleJobJob : jobJobList) {
            ScheduleJob parentScheduleJob = scheduleJobMap.get(scheduleJobJob.getParentJobKey());
            // 1. 补数据实例父实例没有生成,就不去判断父实例状态 2. 周期实例父实例没有生成，就直接设置成实例失败，原因是父实例没有生成
            if (parentScheduleJob == null) {
                if (EScheduleType.NORMAL_SCHEDULE.getType().equals(scheduleJob.getType())) {
                    checkRunInfo.setPass(Boolean.FALSE);
                    checkRunInfo.setStatus(JobCheckStatus.FATHER_NO_CREATED);
                    checkRunInfo.setLogInfo(String.format(JobCheckStatus.FATHER_NO_CREATED.getMsg(), scheduleJob.getJobName(), scheduleJob.getJobId(), scheduleJobJob.getParentJobKey()));
                    return checkRunInfo;
                } else {
                    continue;
                }
            }
            // 判断上游依赖规则
            // 1. 父实例运行完成，可以运行,也就是说父实例状态不影响子任务状态
            // 2. 父实例运行成功，可以运行,也就是说父实例影响子任务状态
            Integer rule = scheduleJobJob.getRule();
            Integer status = parentScheduleJob.getStatus();
            if (RelyRule.RUN_SUCCESS.getType().equals(rule)) {
                Integer jobKeyType = scheduleJobJob.getJobKeyType();
                // 父任务有运行失败的
                if (TaskStatus.FAILED.getStatus().equals(status) || TaskStatus.SUBMITFAILD.getStatus().equals(status) || TaskStatus.PARENTFAILED.getStatus().equals(status)) {
                    checkRunInfo.setPass(Boolean.FALSE);
                    checkRunInfo.setStatus(JobCheckStatus.FATHER_JOB_EXCEPTION);
                    checkRunInfo.setLogInfo(String.format(JobCheckStatus.FATHER_JOB_EXCEPTION.getMsg(), parentScheduleJob.getJobName(), parentScheduleJob.getJobId()));
                    return checkRunInfo;
                }
                // 父实例是冻结(但是这些实例不能是自依赖,自依赖实例是用自己任务的状态判断是否冻结)
                if (TaskStatus.FROZEN.getStatus().equals(status) && !RelyType.SELF_RELIANCE.getType().equals(jobKeyType)) {
                    checkRunInfo.setPass(Boolean.FALSE);
                    checkRunInfo.setStatus(JobCheckStatus.FATHER_JOB_FROZEN);
                    checkRunInfo.setLogInfo(String.format(JobCheckStatus.FATHER_JOB_FROZEN.getMsg(), parentScheduleJob.getJobName(), parentScheduleJob.getJobId()));
                    return checkRunInfo;
                }
                // 父实例是取消
                if (TaskStatus.CANCELED.getStatus().equals(status) || TaskStatus.KILLED.getStatus().equals(status) || TaskStatus.AUTOCANCELED.getStatus().equals(status)) {
                    checkRunInfo.setPass(Boolean.FALSE);
                    checkRunInfo.setStatus(JobCheckStatus.DEPENDENCY_JOB_CANCELED);
                    checkRunInfo.setLogInfo(String.format(JobCheckStatus.DEPENDENCY_JOB_CANCELED.getMsg(), scheduleJob.getJobName(), scheduleJob.getJobId(), parentScheduleJob.getJobName(), parentScheduleJob.getJobId()));
                    return checkRunInfo;
                }
            }
            if (!TaskStatus.FINISHED.getStatus().equals(status) && !TaskStatus.MANUALSUCCESS.getStatus().equals(status)) {
                checkRunInfo.setPass(Boolean.FALSE);
                checkRunInfo.setStatus(JobCheckStatus.FATHER_JOB_NOT_FINISHED);
                checkRunInfo.setLogInfo(JobCheckStatus.FATHER_JOB_NOT_FINISHED.getMsg());
                return checkRunInfo;
            }
        }
    }
    checkRunInfo.setPass(Boolean.TRUE);
    return checkRunInfo;
}