Examples with ScheduleTaskShade com.dtstack.taier.dao.domain.ScheduleTaskShade used on opensource projects

Example 21 with ScheduleTaskShade

use of com.dtstack.taier.dao.domain.ScheduleTaskShade in project Taier by DTStack.

the class UpstreamDependencyHandler method generationJobJobForTask.

@Override
public List<ScheduleJobJob> generationJobJobForTask(ScheduleCorn corn, Date currentDate, String currentJobKey) {
    List<ScheduleJobJob> jobJobList = Lists.newArrayList();
    for (ScheduleTaskShade taskShade : taskShadeList) {
        try {
            String jobKey = getJobKey(taskShade, currentDate);
            // 如果获取不到key，说明是第一天生成实例，则不生成这条边
            if (StringUtils.isBlank(jobKey)) {
                continue;
            }
            ScheduleJobJob scheduleJobJob = new ScheduleJobJob();
            scheduleJobJob.setTenantId(currentTaskShade.getTenantId());
            scheduleJobJob.setJobKey(currentJobKey);
            scheduleJobJob.setParentJobKey(jobKey);
            scheduleJobJob.setJobKeyType(RelyType.UPSTREAM.getType());
            scheduleJobJob.setRule(RelyRule.RUN_SUCCESS.getType());
            scheduleJobJob.setIsDeleted(Deleted.NORMAL.getStatus());
            jobJobList.add(scheduleJobJob);
        } catch (Exception e) {
            LOGGER.error("", e);
        }
    }
    return jobJobList;
}

Example 22 with ScheduleTaskShade

use of com.dtstack.taier.dao.domain.ScheduleTaskShade in project Taier by DTStack.

the class RestartJobRunnable method findCanResetJob.

/**
 * @param jobs
 * @param taskShadeMap
 * @param resumeBatchJobs
 * @return
 */
private List<ScheduleJob> findCanResetJob(List<ScheduleJob> jobs, Map<Long, ScheduleTaskShade> taskShadeMap, Map<String, String> resumeBatchJobs) {
    List<ScheduleJob> canResetList = Lists.newArrayList();
    for (ScheduleJob job : jobs) {
        Integer jobStatus = job.getStatus();
        if (!TaskStatus.canReset(jobStatus)) {
            LOGGER.error("job {} status {}  can not restart ", job.getJobId(), job.getStatus());
            continue;
        }
        ScheduleTaskShade scheduleTaskShade = taskShadeMap.get(job.getTaskId());
        if (scheduleTaskShade == null || Deleted.DELETED.getStatus().equals(scheduleTaskShade.getIsDeleted())) {
            LOGGER.error("cat not find taskShade by taskId:{}", job.getTaskId());
            continue;
        }
        canResetList.add(job);
        // 判断这个任务是否是工作流子任务，如果是需要带上工作流任务
        if (!StringUtils.equals("0", job.getFlowJobId())) {
            ScheduleJob flowJob = scheduleJobService.lambdaQuery().eq(ScheduleJob::getFlowJobId, job.getFlowJobId()).eq(ScheduleJob::getIsDeleted, Deleted.NORMAL.getStatus()).one();
            if (flowJob != null) {
                resumeBatchJobs.put(flowJob.getJobId(), flowJob.getCycTime());
            }
        }
    }
    return canResetList;
}

Example 23 with ScheduleTaskShade

use of com.dtstack.taier.dao.domain.ScheduleTaskShade in project Taier by DTStack.

the class AbstractJobBuilder method buildJob.

@Override
public List<ScheduleJobDetails> buildJob(ScheduleTaskShade scheduleTaskShade, String name, String triggerDay, String beginTime, String endTime, Long fillId, JobSortWorker jobSortWorker) throws Exception {
    // 解析周期信息
    ScheduleCorn corn = ScheduleConfManager.parseFromJson(scheduleTaskShade.getScheduleConf());
    ScheduleConf scheduleConf = corn.getScheduleConf();
    // 校验时间是否符合规范，且获得时间范围
    Pair<Date, Date> triggerRange = getTriggerRange(triggerDay, beginTime, endTime);
    // 获得实例命中的实际范围
    Date startDate = getStartData(scheduleConf, triggerRange, scheduleTaskShade.getTaskId());
    Date endDate = getEndDate(scheduleConf, triggerRange, scheduleTaskShade.getTaskId());
    List<ScheduleJobDetails> jobBuilderBeanList = Lists.newArrayList();
    Date next = corn.isMatch(startDate) ? startDate : corn.next(startDate);
    while (next != null) {
        // 如下下一次执行时间已经在结束时间之后，停止生成实例
        if (next.after(endDate)) {
            break;
        }
        ScheduleJobDetails jobBuilderBean = buildJobBuilderBean(scheduleTaskShade, name, fillId, jobSortWorker, corn, scheduleConf, next, NORMAL_TASK_FLOW_ID);
        if (EScheduleJobType.WORK_FLOW.getVal().equals(scheduleTaskShade.getTaskType())) {
            // 该任务是工作流任务 先生成子任务
            List<ScheduleTaskShade> subTasks = scheduleTaskService.lambdaQuery().eq(ScheduleTaskShade::getFlowId, scheduleTaskShade.getTaskId()).eq(ScheduleTaskShade::getIsDeleted, Deleted.NORMAL.getStatus()).list();
            List<ScheduleJobDetails> flowBean = Lists.newArrayList();
            ScheduleJob scheduleJob = jobBuilderBean.getScheduleJob();
            for (ScheduleTaskShade subTask : subTasks) {
                flowBean.add(buildJobBuilderBean(subTask, name, fillId, jobSortWorker, corn, scheduleConf, next, scheduleJob.getJobId()));
            }
            jobBuilderBean.setFlowBean(flowBean);
        }
        jobBuilderBeanList.add(jobBuilderBean);
        next = corn.next(next);
    }
    return jobBuilderBeanList;
}

Example 24 with ScheduleTaskShade

use of com.dtstack.taier.dao.domain.ScheduleTaskShade in project Taier by DTStack.

the class FillDataJobBuilder method buildFillDataJobGraph.

/**
 * 创建一天的补数据实例
 *
 * @param fillName 补数据名称
 * @param fillId 补数据id
 * @param all all list 所有节点
 * @param run run list 可运行节点
 * @param triggerDay 具体目标天
 * @param beginTime  每天时间范围 开始范围
 * @param endTime 每天时间范围 结束范围
 * @throws Exception
 */
@Transactional(rollbackFor = Exception.class)
private void buildFillDataJobGraph(String fillName, Long fillId, Set<Long> all, Set<Long> run, String triggerDay, String beginTime, String endTime) throws Exception {
    List<Long> allList = Lists.newArrayList(all);
    List<List<Long>> partition = Lists.partition(allList, environmentContext.getJobLimitSize());
    AtomicJobSortWorker sortWorker = new AtomicJobSortWorker();
    for (List<Long> taskKey : partition) {
        jobGraphBuildPool.submit(() -> {
            try {
                List<ScheduleJobDetails> saveList = Lists.newArrayList();
                for (Long taskId : taskKey) {
                    try {
                        ScheduleTaskShade scheduleTaskShade = scheduleTaskService.lambdaQuery().eq(ScheduleTaskShade::getTaskId, taskId).eq(ScheduleTaskShade::getIsDeleted, Deleted.NORMAL.getStatus()).one();
                        if (scheduleTaskShade != null) {
                            List<ScheduleJobDetails> jobBuilderBeanList = Lists.newArrayList();
                            // 非工作流任务子任务
                            if (scheduleTaskShade.getFlowId() == 0) {
                                // 生成补数据实例
                                jobBuilderBeanList = RetryUtil.executeWithRetry(() -> buildJob(scheduleTaskShade, fillName, triggerDay, beginTime, endTime, fillId, sortWorker), environmentContext.getBuildJobErrorRetry(), 200, false);
                            } else {
                                Long flowId = scheduleTaskShade.getFlowId();
                                if (!allList.contains(flowId)) {
                                    // 生成周期实例
                                    jobBuilderBeanList = RetryUtil.executeWithRetry(() -> buildJob(scheduleTaskShade, fillName, triggerDay, beginTime, beginTime, fillId, sortWorker), environmentContext.getBuildJobErrorRetry(), 200, false);
                                }
                            }
                            for (ScheduleJobDetails jobBuilderBean : jobBuilderBeanList) {
                                addMap(run, saveList, taskId, jobBuilderBean);
                            }
                        }
                    } catch (Exception e) {
                        LOGGER.error("taskKey : {} error:", taskId, e);
                    }
                }
                savaFillJob(saveList);
            } catch (Exception e) {
                LOGGER.error("fill error:", e);
            }
        });
    }
}

Example 25 with ScheduleTaskShade

use of com.dtstack.taier.dao.domain.ScheduleTaskShade in project Taier by DTStack.

the class BatchTaskService method deleteTask.

/**
 * 数据开发-删除任务
 *
 * @param taskId    任务id
 * @param tenantId 项目id
 * @param userId    用户id
 * @return
 * @author toutian
 */
@Transactional
public Long deleteTask(Long taskId, Long tenantId, Long userId, String sqlText) {
    final BatchTask batchTask = this.developTaskDao.getOne(taskId);
    if (batchTask == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    // 判断该任务是否有子任务(调用engine接口) 工作流不需要判断
    if (batchTask.getFlowId() == 0) {
        List<TaskGetNotDeleteVO> notDeleteTaskVOS = getChildTasks(taskId);
        if (CollectionUtils.isNotEmpty(notDeleteTaskVOS)) {
            throw new RdosDefineException("(当前任务被其他任务依赖)", ErrorCode.CAN_NOT_DELETE_TASK);
        }
    }
    final ScheduleTaskShade dbTask = this.taskService.findTaskByTaskId(taskId);
    if (batchTask.getFlowId() == 0 && Objects.nonNull(dbTask) && batchTask.getScheduleStatus().intValue() == EScheduleStatus.NORMAL.getVal().intValue()) {
        throw new RdosDefineException("(当前任务未被冻结)", ErrorCode.CAN_NOT_DELETE_TASK);
    }
    if (batchTask.getTaskType().intValue() == EScheduleJobType.WORK_FLOW.getVal()) {
        final List<BatchTask> batchTasks = this.getFlowWorkSubTasks(taskId);
        // 删除所有子任务相关
        batchTasks.forEach(task -> this.deleteTaskInfos(task.getId(), tenantId, userId));
    }
    // 删除工作流中的子任务同时删除被依赖的关系
    if (batchTask.getFlowId() > 0) {
        this.batchTaskTaskService.deleteTaskTaskByParentId(batchTask.getId());
    }
    if (StringUtils.isNotBlank(sqlText)) {
        final BatchTask batchTaskBean = new BatchTask();
        batchTaskBean.setId(batchTask.getFlowId());
        batchTaskBean.setSqlText(sqlText);
        this.developTaskDao.updateSqlText(batchTaskBean);
        LOGGER.info("sqlText 修改成功");
    } else {
        LOGGER.error("deleteTask sqlText is null");
    }
    // 删除任务
    this.deleteTaskInfos(taskId, tenantId, userId);
    return taskId;
}