Examples with BatchTask com.dtstack.taier.dao.domain.BatchTask used on opensource projects

Example 16 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method mapDfs.

/**
 * 图深度遍历
 *
 * @param taskId  任务ID
 * @param set     已经遍历过的节点
 * @param nodeMap 任务完整依赖关系  key：节点  value 节点的所有父节点
 */
private void mapDfs(Long taskId, HashSet<Long> set, Map<Long, List<Long>> nodeMap) {
    HashSet<Long> node = new HashSet<>(set);
    // 判断该节点是否以及存在，如果存在，则证明成环了
    if (set.contains(taskId)) {
        BatchTask task = developTaskDao.getOne(taskId);
        if (Objects.nonNull(task)) {
            throw new RdosDefineException(String.format("%s任务发生依赖闭环", task.getName()));
        }
    }
    node.add(taskId);
    for (Long j : nodeMap.get(taskId)) {
        mapDfs(j, node, nodeMap);
    }
}

Example 17 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method operateIncreCol.

/**
 * 处理增量标识  主要处理两部分 1 处理增量标识字段  2.处理调度依赖
 * @param param
 */
private void operateIncreCol(TaskResourceParam param) {
    final BatchTask task = this.developTaskDao.getOne(param.getId());
    if (StringUtils.isNotEmpty(task.getSqlText())) {
        final JSONObject json = JSON.parseObject(Base64Util.baseDecode(task.getSqlText()));
        json.put("syncModel", param.getSyncModel());
        // 处理增量标示
        operateIncreamColumn(json, param.getSyncModel());
        param.setSqlText(json.toJSONString());
    }
    JSONObject scheduleConf = JSON.parseObject(task.getScheduleConf());
    Integer selfReliance = scheduleConf.getInteger("selfReliance");
    if (param.getSyncModel() == SyncModel.HAS_INCRE_COL.getModel() && !DependencyType.SELF_DEPENDENCY_SUCCESS.getType().equals(selfReliance) && !DependencyType.SELF_DEPENDENCY_END.getType().equals(selfReliance)) {
        scheduleConf.put("selfReliance", DependencyType.SELF_DEPENDENCY_END.getType());
        param.setScheduleConf(scheduleConf.toJSONString());
    }
}

Example 18 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method updateTaskRefResource.

/**
 * 更新任务引用资源
 *
 * @param taskResourceMap
 * @return
 */
@Transactional
public void updateTaskRefResource(final Map<String, Object> taskResourceMap) {
    Preconditions.checkState(taskResourceMap.containsKey("id"), "need param of id");
    Preconditions.checkState(taskResourceMap.containsKey("tenantId"), "need param of tenantId");
    Preconditions.checkState(taskResourceMap.containsKey("createUserId"), "need param of createUserId");
    final Long id = MathUtil.getLongVal(taskResourceMap.get("id"));
    final List<Object> refResourceList = (List<Object>) taskResourceMap.get("refResource");
    final BatchTask task = getOneWithError(id);
    // 删除旧的资源
    batchTaskResourceService.deleteByTaskId(task.getId(), ResourceRefType.DEPENDENCY_RES.getType());
    // 添加新的关联资源
    if (CollectionUtils.isNotEmpty(refResourceList)) {
        final List<Long> refResourceIdList = Lists.newArrayList();
        refResourceList.forEach(tmpId -> refResourceIdList.add(MathUtil.getLongVal(tmpId)));
        this.batchTaskResourceService.save(task, refResourceIdList, ResourceRefType.DEPENDENCY_RES.getType());
    }
}

Example 19 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method checkTaskAndSaveVersion.

/**
 * 检查要发布的任务并保存版本信息
 *
 * @param task 任务信息
 * @param tenantId 项目id
 * @param userId 用户id
 * @param publishDesc 发布描述
 * @param isRoot 是否是管理员
 * @param ignoreCheck 是否忽略检查
 * @return 检查结果
 */
private TaskCheckResultVO checkTaskAndSaveVersion(BatchTask task, Long tenantId, Long userId, String publishDesc, Boolean isRoot, Boolean ignoreCheck) {
    TaskCheckResultVO checkVo = new TaskCheckResultVO();
    checkVo.setErrorSign(PublishTaskStatusEnum.NOMAL.getType());
    checkTaskCanSubmit(task);
    task.setSubmitStatus(ESubmitStatus.SUBMIT.getStatus());
    task.setGmtModified(Timestamp.valueOf(LocalDateTime.now()));
    final BatchTaskVersion version = new BatchTaskVersion();
    version.setCreateUserId(userId);
    String versionSqlText = StringUtils.EMPTY;
    if (EScheduleJobType.SPARK_SQL.getVal().intValue() == task.getTaskType().intValue()) {
        // 语法检测
        List<BatchTaskParam> taskParamsToReplace = batchTaskParamService.getTaskParam(task.getId());
        versionSqlText = this.jobParamReplace.paramReplace(task.getSqlText(), taskParamsToReplace, this.sdf.format(new Date()));
        // 避免重复校验
        CheckSyntaxResult syntaxResult = batchSqlExeService.processSqlText(task.getTenantId(), task.getTaskType(), versionSqlText);
        if (!syntaxResult.getCheckResult()) {
            checkVo.setErrorSign(PublishTaskStatusEnum.CHECKSYNTAXERROR.getType());
            checkVo.setErrorMessage(syntaxResult.getMessage());
            return checkVo;
        }
    } else if (EScheduleJobType.SYNC.getVal().intValue() == task.getTaskType().intValue()) {
        if (StringUtils.isNotEmpty(task.getSqlText())) {
            final JSONObject jsonTask = JSON.parseObject(Base64Util.baseDecode(task.getSqlText()));
            Integer createModelType = Integer.valueOf(jsonTask.getString("createModel"));
            JSONObject job = jsonTask.getJSONObject("job");
            if (Objects.isNull(job)) {
                throw new RdosDefineException(String.format("数据同步任务：%s 未配置", task.getName()));
            }
            // 检测job格式
            SyncJobCheck.checkJobFormat(job.toJSONString(), createModelType);
            versionSqlText = jsonTask.getString("job");
        }
    }
    version.setSqlText(versionSqlText);
    version.setOriginSql(task.getSqlText());
    version.setTenantId(task.getTenantId());
    version.setTaskId(task.getId());
    // 任务的版本号
    version.setVersion(task.getVersion());
    version.setTaskParams(task.getTaskParams());
    version.setScheduleConf(task.getScheduleConf());
    version.setScheduleStatus(task.getScheduleStatus());
    version.setGmtModified(task.getGmtModified());
    String dependencyTaskIds = StringUtils.EMPTY;
    final List<BatchTaskTask> taskTasks = this.batchTaskTaskService.getAllParentTask(task.getId());
    if (CollectionUtils.isNotEmpty(taskTasks)) {
        List<Map<String, Object>> parentTasks = taskTasks.stream().map(taskTask -> {
            Map<String, Object> map = Maps.newHashMap();
            map.put("parentTaskId", taskTask.getParentTaskId());
            return map;
        }).collect(Collectors.toList());
        dependencyTaskIds = JSON.toJSONString(parentTasks);
    }
    version.setDependencyTaskIds(dependencyTaskIds);
    version.setPublishDesc(null == publishDesc ? "" : publishDesc);
    // 插入一条记录信息
    batchTaskVersionService.insert(version);
    task.setVersion(version.getId().intValue());
    return checkVo;
}

Example 20 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method publishBatchTaskInfo.

/**
 * 批量发布任务至engine
 * @param publishTask 要发布的task集合
 * @param tenantId 项目id
 * @param userId 用户id
 * @param publishDesc 发布描述
 * @param isRoot 是否是管理员
 * @param ignoreCheck 忽略检查
 * @return 发布结果
 */
public TaskCheckResultVO publishBatchTaskInfo(BatchTask publishTask, Long tenantId, Long userId, String publishDesc, Boolean isRoot, Boolean ignoreCheck) {
    // 判断任务责任人是否存在 如果任务责任人不存在或无权限 不允许提交
    User user = userService.getById(publishTask.getCreateUserId());
    if (user == null) {
        throw new RdosDefineException(String.format("%s任务责任人在数栈中不存在", publishTask.getName()));
    }
    TaskCheckResultVO checkResultVO = new TaskCheckResultVO();
    checkResultVO.setErrorSign(PublishTaskStatusEnum.NOMAL.getType());
    // 检查任务是否可以发布并记录版本信息
    TaskCheckResultVO resultVO = checkTaskAndSaveVersion(publishTask, tenantId, userId, publishDesc, isRoot, ignoreCheck);
    if (!PublishTaskStatusEnum.NOMAL.getType().equals(resultVO.getErrorSign())) {
        // 做一下优化 如果是工作流任务的话 把任务名称打印出来
        if (publishTask.getFlowId() > 0) {
            resultVO.setErrorMessage(String.format("任务:%s提交失败，原因是:%s", publishTask.getName(), resultVO.getErrorMessage()));
        }
        return resultVO;
    }
    // 发布任务中所有的依赖关系
    List<Long> parentTaskIds = Lists.newArrayList();
    ScheduleTaskShadeDTO scheduleTaskShadeDTO = buildScheduleTaskShadeDTO(publishTask, parentTaskIds);
    // 提交任务参数信息并保存任务记录和更新任务状态
    try {
        BatchTask batchTask = getOneWithError(scheduleTaskShadeDTO.getTaskId());
        String extraInfo = this.batchJobService.getExtraInfo(batchTask, userId, null);
        scheduleTaskShadeDTO.setExtraInfo(extraInfo);
        // 无异常保存一条任务记录并更新任务状态
        saveRecordAndUpdateSubmitStatus(batchTask, tenantId, userId, TaskOperateType.COMMIT.getType(), ESubmitStatus.SUBMIT.getStatus());
    } catch (Exception e) {
        LOGGER.error("send task error {} ", scheduleTaskShadeDTO.getTaskId(), e);
        throw new RdosDefineException(String.format("任务提交异常：%s", e.getMessage()), e);
    }
    SavaTaskDTO savaTaskDTO = new SavaTaskDTO();
    savaTaskDTO.setScheduleTaskShadeDTO(scheduleTaskShadeDTO);
    savaTaskDTO.setParentTaskIdList(parentTaskIds);
    // 批量发布任务
    this.taskService.saveTask(savaTaskDTO);
    LOGGER.info("待发布任务参数提交完毕");
    return checkResultVO;
}