Examples with BatchTaskParam com.dtstack.taier.dao.domain.BatchTaskParam used on opensource projects

Example 1 with BatchTaskParam

use of com.dtstack.taier.dao.domain.BatchTaskParam in project Taier by DTStack.

the class BatchTaskParamService method getTaskParam.

public List<BatchTaskParam> getTaskParam(final long taskId) {
    List<BatchTaskParam> taskParams = developTaskParamDao.listByTaskId(taskId);
    // 特殊处理 TaskParam 系统参数
    for (BatchTaskParam taskParamShade : taskParams) {
        if (!EParamType.SYS_TYPE.getType().equals(taskParamShade.getType())) {
            continue;
        }
        // 将 command 属性设置为系统表的 command
        BatchSysParameter sysParameter = batchSysParamService.getBatchSysParamByName(taskParamShade.getParamName());
        taskParamShade.setParamCommand(sysParameter.getParamCommand());
    }
    return taskParams;
}

Example 2 with BatchTaskParam

use of com.dtstack.taier.dao.domain.BatchTaskParam in project Taier by DTStack.

the class BatchTaskParamService method checkParams.

/**
 * 校验任务中的 系统参数 和 自定义参数
 * @param jobContent SQL内容
 * @param parameterSet 任务参数
 */
public void checkParams(final String jobContent, final List parameterSet) {
    // 校验任务参数不能为空参数
    if (CollectionUtils.isNotEmpty(parameterSet)) {
        for (Object paramObj : parameterSet) {
            BatchTaskParam batchTaskParam = PublicUtil.objectToObject(paramObj, BatchTaskParam.class);
            if (batchTaskParam != null) {
                if (StringUtils.isBlank(batchTaskParam.getParamCommand()) || "$[]".equalsIgnoreCase(batchTaskParam.getParamCommand())) {
                    throw new RdosDefineException("自定义参数赋值不能为空");
                }
            }
        }
    }
    String jobStr = jobContent;
    if (StringUtils.isBlank(jobStr)) {
        return;
    }
    // 校验任务参数时，先清除sql中的注释
    String sqlWithoutComments = this.batchSqlExeService.removeComment(jobStr);
    if (StringUtils.isNotEmpty(sqlWithoutComments)) {
        sqlWithoutComments = sqlWithoutComments.replaceAll("\\s*", "");
    }
    // 校验任务参数时，先删除 数据同步任务 配置项
    if (sqlWithoutComments.contains(FormNames.HBASE_CONFIG) || sqlWithoutComments.contains(FormNames.HADOOP_CONFIG) || sqlWithoutComments.contains(FormNames.KERBEROS_CONFIG)) {
        sqlWithoutComments = removeConfig(sqlWithoutComments);
    }
    // 正则解析SQL中的 系统参数 和 自定义参数
    Matcher matcher = PARAM_REGEX_PATTERN.matcher(sqlWithoutComments);
    if (matcher.find()) {
        if (CollectionUtils.isEmpty(parameterSet)) {
            LOGGER.error("jobContent:{}", jobContent);
            throw new RdosDefineException(ErrorCode.TASK_PARAM_CONTENT_NOT_NULL);
        }
    }
}

Example 3 with BatchTaskParam

use of com.dtstack.taier.dao.domain.BatchTaskParam in project Taier by DTStack.

the class BatchTaskService method buildScheduleTaskShadeDTO.

/**
 * 构建一个要发布到engine的任务DTO {@link ScheduleTaskShadeDTO}
 * @param batchTask 要发布的任务集合
 * @param parentTaskIds 父任务的id
 * @return 调度任务DTO
 */
private ScheduleTaskShadeDTO buildScheduleTaskShadeDTO(final BatchTask batchTask, List<Long> parentTaskIds) {
    if (batchTask.getId() <= 0) {
        // 只有异常情况才会走到该逻辑
        throw new RdosDefineException("batchTask id can't be 0", ErrorCode.SERVER_EXCEPTION);
    }
    final long taskId = batchTask.getId();
    // 清空任务关联的batch_task_param, task_resource, task_task 表信息
    this.batchTaskParamShadeService.clearDataByTaskId(taskId);
    this.batchTaskResourceShadeService.clearDataByTaskId(taskId);
    final List<BatchTaskParam> batchTaskParamList = this.batchTaskParamService.getTaskParam(batchTask.getId());
    // 查询出任务所有的关联的资源(运行主体资源和依赖引用资源)
    final List<BatchTaskResource> batchTaskResourceList = this.batchTaskResourceService.getTaskResources(batchTask.getId(), null);
    List<Long> parentTaskList = this.batchTaskTaskService.getAllParentTaskId(batchTask.getId());
    parentTaskIds.addAll(parentTaskList);
    if (!CollectionUtils.isEmpty(batchTaskResourceList)) {
        this.batchTaskResourceShadeService.saveTaskResource(batchTaskResourceList);
    }
    // 保存batch_task_shade
    final ScheduleTaskShadeDTO scheduleTaskShadeDTO = new ScheduleTaskShadeDTO();
    BeanUtils.copyProperties(batchTask, scheduleTaskShadeDTO);
    scheduleTaskShadeDTO.setTaskId(batchTask.getId());
    scheduleTaskShadeDTO.setScheduleStatus(EScheduleStatus.NORMAL.getVal());
    if (!CollectionUtils.isEmpty(batchTaskParamList)) {
        this.batchTaskParamShadeService.saveTaskParam(batchTaskParamList);
    } else {
        scheduleTaskShadeDTO.setTaskParams("");
    }
    return scheduleTaskShadeDTO;
}

Example 4 with BatchTaskParam

use of com.dtstack.taier.dao.domain.BatchTaskParam in project Taier by DTStack.

the class BatchTaskService method checkTaskAndSaveVersion.

/**
 * 检查要发布的任务并保存版本信息
 *
 * @param task 任务信息
 * @param tenantId 项目id
 * @param userId 用户id
 * @param publishDesc 发布描述
 * @param isRoot 是否是管理员
 * @param ignoreCheck 是否忽略检查
 * @return 检查结果
 */
private TaskCheckResultVO checkTaskAndSaveVersion(BatchTask task, Long tenantId, Long userId, String publishDesc, Boolean isRoot, Boolean ignoreCheck) {
    TaskCheckResultVO checkVo = new TaskCheckResultVO();
    checkVo.setErrorSign(PublishTaskStatusEnum.NOMAL.getType());
    checkTaskCanSubmit(task);
    task.setSubmitStatus(ESubmitStatus.SUBMIT.getStatus());
    task.setGmtModified(Timestamp.valueOf(LocalDateTime.now()));
    final BatchTaskVersion version = new BatchTaskVersion();
    version.setCreateUserId(userId);
    String versionSqlText = StringUtils.EMPTY;
    if (EScheduleJobType.SPARK_SQL.getVal().intValue() == task.getTaskType().intValue()) {
        // 语法检测
        List<BatchTaskParam> taskParamsToReplace = batchTaskParamService.getTaskParam(task.getId());
        versionSqlText = this.jobParamReplace.paramReplace(task.getSqlText(), taskParamsToReplace, this.sdf.format(new Date()));
        // 避免重复校验
        CheckSyntaxResult syntaxResult = batchSqlExeService.processSqlText(task.getTenantId(), task.getTaskType(), versionSqlText);
        if (!syntaxResult.getCheckResult()) {
            checkVo.setErrorSign(PublishTaskStatusEnum.CHECKSYNTAXERROR.getType());
            checkVo.setErrorMessage(syntaxResult.getMessage());
            return checkVo;
        }
    } else if (EScheduleJobType.SYNC.getVal().intValue() == task.getTaskType().intValue()) {
        if (StringUtils.isNotEmpty(task.getSqlText())) {
            final JSONObject jsonTask = JSON.parseObject(Base64Util.baseDecode(task.getSqlText()));
            Integer createModelType = Integer.valueOf(jsonTask.getString("createModel"));
            JSONObject job = jsonTask.getJSONObject("job");
            if (Objects.isNull(job)) {
                throw new RdosDefineException(String.format("数据同步任务：%s 未配置", task.getName()));
            }
            // 检测job格式
            SyncJobCheck.checkJobFormat(job.toJSONString(), createModelType);
            versionSqlText = jsonTask.getString("job");
        }
    }
    version.setSqlText(versionSqlText);
    version.setOriginSql(task.getSqlText());
    version.setTenantId(task.getTenantId());
    version.setTaskId(task.getId());
    // 任务的版本号
    version.setVersion(task.getVersion());
    version.setTaskParams(task.getTaskParams());
    version.setScheduleConf(task.getScheduleConf());
    version.setScheduleStatus(task.getScheduleStatus());
    version.setGmtModified(task.getGmtModified());
    String dependencyTaskIds = StringUtils.EMPTY;
    final List<BatchTaskTask> taskTasks = this.batchTaskTaskService.getAllParentTask(task.getId());
    if (CollectionUtils.isNotEmpty(taskTasks)) {
        List<Map<String, Object>> parentTasks = taskTasks.stream().map(taskTask -> {
            Map<String, Object> map = Maps.newHashMap();
            map.put("parentTaskId", taskTask.getParentTaskId());
            return map;
        }).collect(Collectors.toList());
        dependencyTaskIds = JSON.toJSONString(parentTasks);
    }
    version.setDependencyTaskIds(dependencyTaskIds);
    version.setPublishDesc(null == publishDesc ? "" : publishDesc);
    // 插入一条记录信息
    batchTaskVersionService.insert(version);
    task.setVersion(version.getId().intValue());
    return checkVo;
}

Example 5 with BatchTaskParam

use of com.dtstack.taier.dao.domain.BatchTaskParam in project Taier by DTStack.

the class BatchTaskParamService method buildBatchTaskParams.

public List<BatchTaskParam> buildBatchTaskParams(final long taskId, final List<BatchParamDTO> batchParamDTOS) {
    final List<BatchTaskParam> saves = new ArrayList<>(batchParamDTOS.size());
    for (final BatchParamDTO tmp : batchParamDTOS) {
        if (StringUtils.isBlank(tmp.getParamCommand())) {
            throw new RdosDefineException("自定义参数赋值不能为空");
        }
        BatchTaskParam batchTaskParam = new BatchTaskParam();
        batchTaskParam.setTaskId(taskId);
        batchTaskParam.setType(tmp.getType());
        batchTaskParam.setParamName(tmp.getParamName());
        batchTaskParam.setParamCommand(tmp.getParamCommand());
        saves.add(this.addOrUpdate(batchTaskParam));
    }
    return saves;
}