Examples with ScheduleJobDetails com.dtstack.taier.scheduler.server.ScheduleJobDetails used on opensource projects

Example 6 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class CycleJobScheduler method listExecJob.

@Override
protected List<ScheduleJobDetails> listExecJob(Long startSort, String nodeAddress, Boolean isEq) {
    List<ScheduleJob> scheduleJobList = scheduleJobService.listCycleJob(startSort, nodeAddress, getScheduleType().getType(), isEq, JobPhaseStatus.CREATE.getCode());
    List<String> jobKeys = scheduleJobList.stream().map(ScheduleJob::getJobKey).collect(Collectors.toList());
    List<ScheduleJobJob> scheduleJobJobList = scheduleJobJobService.listByJobKeys(jobKeys);
    Map<String, List<ScheduleJobJob>> jobJobMap = scheduleJobJobList.stream().collect(Collectors.groupingBy(ScheduleJobJob::getJobKey));
    List<ScheduleJobDetails> scheduleJobDetailsList = new ArrayList<>(scheduleJobList.size());
    for (ScheduleJob scheduleJob : scheduleJobList) {
        ScheduleJobDetails scheduleJobDetails = new ScheduleJobDetails();
        scheduleJobDetails.setScheduleJob(scheduleJob);
        scheduleJobDetails.setJobJobList(jobJobMap.get(scheduleJob.getJobKey()));
        scheduleJobDetailsList.add(scheduleJobDetails);
    }
    return scheduleJobDetailsList;
}

Example 7 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class ScheduleJobService method insertJobList.

/**
 * 批量插入周期实例 jobSize 在负载均衡时 区分 scheduleType（正常调度 和 补数据）
 *
 * @param jobBuilderBeanCollection 实例集合
 * @param scheduleType 调度类型 正常调度 和 补数据
 */
@Transactional(rollbackFor = Exception.class)
public Long insertJobList(Collection<ScheduleJobDetails> jobBuilderBeanCollection, Integer scheduleType) {
    if (CollectionUtils.isEmpty(jobBuilderBeanCollection)) {
        return null;
    }
    Iterator<ScheduleJobDetails> batchJobIterator = jobBuilderBeanCollection.iterator();
    // count%20 为一批
    // 1: 批量插入BatchJob
    // 2: 批量插入BatchJobJobList
    int count = 0;
    int jobBatchSize = environmentContext.getBatchJobInsertSize();
    int jobJobBatchSize = environmentContext.getBatchJobJobInsertSize();
    Long minJobId = null;
    List<ScheduleJob> jobWaitForSave = Lists.newArrayList();
    List<ScheduleJobJob> jobJobWaitForSave = Lists.newArrayList();
    Map<String, Integer> nodeJobSize = computeJobSizeForNode(jobBuilderBeanCollection.size(), scheduleType);
    for (Map.Entry<String, Integer> nodeJobSizeEntry : nodeJobSize.entrySet()) {
        String nodeAddress = nodeJobSizeEntry.getKey();
        int nodeSize = nodeJobSizeEntry.getValue();
        final int finalBatchNodeSize = nodeSize;
        while (nodeSize > 0 && batchJobIterator.hasNext()) {
            nodeSize--;
            count++;
            ScheduleJobDetails jobBuilderBean = batchJobIterator.next();
            ScheduleJob scheduleJob = jobBuilderBean.getScheduleJob();
            scheduleJob.setNodeAddress(nodeAddress);
            jobWaitForSave.add(scheduleJob);
            jobJobWaitForSave.addAll(jobBuilderBean.getJobJobList());
            LOGGER.debug("insertJobList count:{} batchJobs:{} finalBatchNodeSize:{}", count, jobBuilderBeanCollection.size(), finalBatchNodeSize);
            if (count % jobBatchSize == 0 || count == (jobBuilderBeanCollection.size() - 1) || jobJobWaitForSave.size() > jobJobBatchSize) {
                minJobId = persistJobs(jobWaitForSave, jobJobWaitForSave, minJobId, jobJobBatchSize);
                LOGGER.info("insertJobList count:{} batchJobs:{} finalBatchNodeSize:{} jobJobSize:{}", count, jobBuilderBeanCollection.size(), finalBatchNodeSize, jobJobWaitForSave.size());
            }
        }
        LOGGER.info("insertJobList count:{} batchJobs:{} finalBatchNodeSize:{}", count, jobBuilderBeanCollection.size(), finalBatchNodeSize);
        // 结束前persist一次，flush所有jobs
        minJobId = persistJobs(jobWaitForSave, jobJobWaitForSave, minJobId, jobJobBatchSize);
    }
    return minJobId;
}

Example 8 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class OperatorRecordJobScheduler method listExecJob.

@Override
protected List<ScheduleJobDetails> listExecJob(Long startSort, String nodeAddress, Boolean isEq) {
    List<ScheduleJobOperatorRecord> records = scheduleJobOperatorRecordService.listOperatorRecord(startSort, nodeAddress, getOperatorType().getType(), isEq);
    if (CollectionUtils.isNotEmpty(records)) {
        Set<String> jobIds = records.stream().map(ScheduleJobOperatorRecord::getJobId).collect(Collectors.toSet());
        List<ScheduleJob> scheduleJobList = getScheduleJob(jobIds);
        if (CollectionUtils.isNotEmpty(scheduleJobList)) {
            List<String> jodExecIds = scheduleJobList.stream().map(ScheduleJob::getJobId).collect(Collectors.toList());
            if (jobIds.size() != scheduleJobList.size()) {
                // 过滤出来已经提交运行的实例，删除操作记录
                List<String> deleteJobIdList = jobIds.stream().filter(jobId -> !jodExecIds.contains(jobId)).collect(Collectors.toList());
                removeOperatorRecord(deleteJobIdList);
            }
            List<String> jobKeys = scheduleJobList.stream().map(ScheduleJob::getJobKey).collect(Collectors.toList());
            List<ScheduleJobJob> scheduleJobJobList = scheduleJobJobService.listByJobKeys(jobKeys);
            Map<String, List<ScheduleJobJob>> jobJobMap = scheduleJobJobList.stream().collect(Collectors.groupingBy(ScheduleJobJob::getJobKey));
            List<ScheduleJobDetails> scheduleJobDetailsList = new ArrayList<>(scheduleJobList.size());
            for (ScheduleJob scheduleJob : scheduleJobList) {
                ScheduleJobDetails scheduleJobDetails = new ScheduleJobDetails();
                scheduleJobDetails.setScheduleJob(scheduleJob);
                scheduleJobDetails.setJobJobList(jobJobMap.get(scheduleJob.getJobKey()));
                scheduleJobDetailsList.add(scheduleJobDetails);
            }
            return scheduleJobDetailsList;
        } else {
            removeOperatorRecord(Lists.newArrayList(jobIds));
        }
    }
    return Lists.newArrayList();
}

Example 9 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class AbstractJobBuilder method buildJob.

@Override
public List<ScheduleJobDetails> buildJob(ScheduleTaskShade scheduleTaskShade, String name, String triggerDay, String beginTime, String endTime, Long fillId, JobSortWorker jobSortWorker) throws Exception {
    // 解析周期信息
    ScheduleCorn corn = ScheduleConfManager.parseFromJson(scheduleTaskShade.getScheduleConf());
    ScheduleConf scheduleConf = corn.getScheduleConf();
    // 校验时间是否符合规范，且获得时间范围
    Pair<Date, Date> triggerRange = getTriggerRange(triggerDay, beginTime, endTime);
    // 获得实例命中的实际范围
    Date startDate = getStartData(scheduleConf, triggerRange, scheduleTaskShade.getTaskId());
    Date endDate = getEndDate(scheduleConf, triggerRange, scheduleTaskShade.getTaskId());
    List<ScheduleJobDetails> jobBuilderBeanList = Lists.newArrayList();
    Date next = corn.isMatch(startDate) ? startDate : corn.next(startDate);
    while (next != null) {
        // 如下下一次执行时间已经在结束时间之后，停止生成实例
        if (next.after(endDate)) {
            break;
        }
        ScheduleJobDetails jobBuilderBean = buildJobBuilderBean(scheduleTaskShade, name, fillId, jobSortWorker, corn, scheduleConf, next, NORMAL_TASK_FLOW_ID);
        if (EScheduleJobType.WORK_FLOW.getVal().equals(scheduleTaskShade.getTaskType())) {
            // 该任务是工作流任务 先生成子任务
            List<ScheduleTaskShade> subTasks = scheduleTaskService.lambdaQuery().eq(ScheduleTaskShade::getFlowId, scheduleTaskShade.getTaskId()).eq(ScheduleTaskShade::getIsDeleted, Deleted.NORMAL.getStatus()).list();
            List<ScheduleJobDetails> flowBean = Lists.newArrayList();
            ScheduleJob scheduleJob = jobBuilderBean.getScheduleJob();
            for (ScheduleTaskShade subTask : subTasks) {
                flowBean.add(buildJobBuilderBean(subTask, name, fillId, jobSortWorker, corn, scheduleConf, next, scheduleJob.getJobId()));
            }
            jobBuilderBean.setFlowBean(flowBean);
        }
        jobBuilderBeanList.add(jobBuilderBean);
        next = corn.next(next);
    }
    return jobBuilderBeanList;
}

Example 10 with ScheduleJobDetails

use of com.dtstack.taier.scheduler.server.ScheduleJobDetails in project Taier by DTStack.

the class AbstractJobBuilder method buildJobBuilderBean.

/**
 * 构建JobBuilderBean
 *
 * @param scheduleTaskShade 需要被构建的任务
 * @param name 实例名称
 * @param fillId 补数据id
 * @param jobSortWorker 排序器
 * @param corn 周期实例
 * @param scheduleConf 调度配置
 * @param currentData 当前时间
 * @return
 */
private ScheduleJobDetails buildJobBuilderBean(ScheduleTaskShade scheduleTaskShade, String name, Long fillId, JobSortWorker jobSortWorker, ScheduleCorn corn, ScheduleConf scheduleConf, Date currentData, String flowJobId) {
    String triggerTime = DateUtil.getDate(currentData, DateUtil.STANDARD_DATETIME_FORMAT);
    String cycTime = DateUtil.getTimeStrWithoutSymbol(triggerTime);
    String jobKey = JobKeyUtils.generateJobKey(getKeyPreStr(name), scheduleTaskShade.getTaskId(), cycTime);
    // 实例
    ScheduleJob scheduleJob = new ScheduleJob();
    scheduleJob.setTenantId(scheduleTaskShade.getTenantId());
    scheduleJob.setJobId(actionService.generateUniqueSign());
    scheduleJob.setJobKey(jobKey);
    scheduleJob.setJobName(getName(scheduleTaskShade, name, cycTime));
    scheduleJob.setTaskId(scheduleTaskShade.getTaskId());
    scheduleJob.setCreateUserId(scheduleTaskShade.getCreateUserId());
    scheduleJob.setIsDeleted(Deleted.NORMAL.getStatus());
    scheduleJob.setType(getType());
    scheduleJob.setIsRestart(Restarted.NORMAL.getStatus());
    scheduleJob.setCycTime(cycTime);
    scheduleJob.setDependencyType(scheduleConf.getSelfReliance());
    scheduleJob.setFlowJobId(flowJobId);
    scheduleJob.setPeriodType(scheduleConf.getPeriodType());
    scheduleJob.setStatus(TaskStatus.UNSUBMIT.getStatus());
    scheduleJob.setTaskType(scheduleTaskShade.getTaskType());
    scheduleJob.setFillId(fillId);
    scheduleJob.setMaxRetryNum(scheduleConf.getMaxRetryNum());
    scheduleJob.setVersionId(scheduleTaskShade.getVersionId());
    scheduleJob.setComputeType(scheduleTaskShade.getComputeType());
    scheduleJob.setNextCycTime(DateUtil.getDate(corn.next(currentData), DateUtil.STANDARD_DATETIME_FORMAT));
    scheduleJob.setJobExecuteOrder(JobExecuteOrderUtil.buildJobExecuteOrder(cycTime, jobSortWorker.getSort()));
    // 获得依赖
    List<ScheduleJobJob> jobJobList = Lists.newArrayList();
    DependencyHandler dependencyHandler = dependencyManager.getDependencyHandler(getKeyPreStr(name), scheduleTaskShade, corn);
    while (dependencyHandler != null) {
        jobJobList.addAll(dependencyHandler.generationJobJobForTask(corn, currentData, jobKey));
        dependencyHandler = dependencyHandler.next();
    }
    ScheduleJobDetails jobBuilderBean = new ScheduleJobDetails();
    jobBuilderBean.setJobJobList(jobJobList);
    jobBuilderBean.setScheduleJob(scheduleJob);
    return jobBuilderBean;
}