Examples with CheckSyntaxResult com.dtstack.taier.develop.dto.devlop.CheckSyntaxResult used on opensource projects

Example 1 with CheckSyntaxResult

use of com.dtstack.taier.develop.dto.devlop.CheckSyntaxResult in project Taier by DTStack.

the class BatchHadoopJobExeService method readyForTaskStartTrigger.

/**
 * 构建 exeArgs、sqlText、taskParams
 * @param actionParam
 * @param tenantId
 * @param batchTask
 * @param taskParamsToReplace
 * @throws Exception
 */
@Override
public void readyForTaskStartTrigger(Map<String, Object> actionParam, Long tenantId, BatchTask batchTask, List<BatchTaskParamShade> taskParamsToReplace) throws Exception {
    String sql = batchTask.getSqlText() == null ? "" : batchTask.getSqlText();
    String taskParams = batchTask.getTaskParams();
    if (EScheduleJobType.SPARK_SQL.getVal().equals(batchTask.getTaskType())) {
        // Spark SQL任务默认在前面加上建表的类型
        sql = String.format("set hive.default.fileformat=%s;\n ", environmentContext.getCreateTableType()) + sql;
        batchTaskParamService.checkParams(sql, taskParamsToReplace);
        // 构建运行的SQL
        CheckSyntaxResult result = batchSqlExeService.processSqlText(tenantId, batchTask.getTaskType(), sql);
        sql = result.getSql();
    } else if (EScheduleJobType.SYNC.getVal().equals(batchTask.getTaskType())) {
        JSONObject syncJob = JSON.parseObject(Base64Util.baseDecode(batchTask.getSqlText()));
        taskParams = replaceSyncParll(taskParams, parseSyncChannel(syncJob));
        String job = syncJob.getString("job");
        // 向导模式根据job中的sourceId填充数据源信息，保证每次运行取到最新的连接信息
        job = datasourceService.setJobDataSourceInfo(job, tenantId, syncJob.getIntValue("createModel"));
        batchTaskParamService.checkParams(batchTaskParamService.checkSyncJobParams(job), taskParamsToReplace);
        actionParam.put("job", job);
    }
    actionParam.put("sqlText", sql);
    actionParam.put("taskParams", taskParams);
}

Example 2 with CheckSyntaxResult

use of com.dtstack.taier.develop.dto.devlop.CheckSyntaxResult in project Taier by DTStack.

the class BatchSqlExeService method processSqlText.

/**
 * 处理自定义函数 和 构建真正运行的SQL
 *
 * @param tenantId
 * @param taskType
 * @param sqlText
 * @return
 */
public CheckSyntaxResult processSqlText(final Long tenantId, Integer taskType, final String sqlText) {
    CheckSyntaxResult result = new CheckSyntaxResult();
    TenantComponent tenantEngine = this.developTenantComponentService.getByTenantAndEngineType(tenantId, taskType);
    Preconditions.checkNotNull(tenantEngine, String.format("tenantEngine %d not support task type %d", tenantId, taskType));
    ISqlExeService sqlExeService = this.multiEngineServiceFactory.getSqlExeService(taskType);
    // 处理自定义函数
    String sqlPlus = buildCustomFunctionSparkSql(sqlText, tenantId, taskType);
    // 构建真正运行的SQL，去掉注释，加上use db 同时格式化SQL
    String sqls = sqlExeService.process(sqlPlus, tenantEngine.getComponentIdentity());
    result.setSql(sqls);
    result.setCheckResult(true);
    return result;
}

Example 3 with CheckSyntaxResult

use of com.dtstack.taier.develop.dto.devlop.CheckSyntaxResult in project Taier by DTStack.

the class BatchTaskService method checkTaskAndSaveVersion.

/**
 * 检查要发布的任务并保存版本信息
 *
 * @param task 任务信息
 * @param tenantId 项目id
 * @param userId 用户id
 * @param publishDesc 发布描述
 * @param isRoot 是否是管理员
 * @param ignoreCheck 是否忽略检查
 * @return 检查结果
 */
private TaskCheckResultVO checkTaskAndSaveVersion(BatchTask task, Long tenantId, Long userId, String publishDesc, Boolean isRoot, Boolean ignoreCheck) {
    TaskCheckResultVO checkVo = new TaskCheckResultVO();
    checkVo.setErrorSign(PublishTaskStatusEnum.NOMAL.getType());
    checkTaskCanSubmit(task);
    task.setSubmitStatus(ESubmitStatus.SUBMIT.getStatus());
    task.setGmtModified(Timestamp.valueOf(LocalDateTime.now()));
    final BatchTaskVersion version = new BatchTaskVersion();
    version.setCreateUserId(userId);
    String versionSqlText = StringUtils.EMPTY;
    if (EScheduleJobType.SPARK_SQL.getVal().intValue() == task.getTaskType().intValue()) {
        // 语法检测
        List<BatchTaskParam> taskParamsToReplace = batchTaskParamService.getTaskParam(task.getId());
        versionSqlText = this.jobParamReplace.paramReplace(task.getSqlText(), taskParamsToReplace, this.sdf.format(new Date()));
        // 避免重复校验
        CheckSyntaxResult syntaxResult = batchSqlExeService.processSqlText(task.getTenantId(), task.getTaskType(), versionSqlText);
        if (!syntaxResult.getCheckResult()) {
            checkVo.setErrorSign(PublishTaskStatusEnum.CHECKSYNTAXERROR.getType());
            checkVo.setErrorMessage(syntaxResult.getMessage());
            return checkVo;
        }
    } else if (EScheduleJobType.SYNC.getVal().intValue() == task.getTaskType().intValue()) {
        if (StringUtils.isNotEmpty(task.getSqlText())) {
            final JSONObject jsonTask = JSON.parseObject(Base64Util.baseDecode(task.getSqlText()));
            Integer createModelType = Integer.valueOf(jsonTask.getString("createModel"));
            JSONObject job = jsonTask.getJSONObject("job");
            if (Objects.isNull(job)) {
                throw new RdosDefineException(String.format("数据同步任务：%s 未配置", task.getName()));
            }
            // 检测job格式
            SyncJobCheck.checkJobFormat(job.toJSONString(), createModelType);
            versionSqlText = jsonTask.getString("job");
        }
    }
    version.setSqlText(versionSqlText);
    version.setOriginSql(task.getSqlText());
    version.setTenantId(task.getTenantId());
    version.setTaskId(task.getId());
    // 任务的版本号
    version.setVersion(task.getVersion());
    version.setTaskParams(task.getTaskParams());
    version.setScheduleConf(task.getScheduleConf());
    version.setScheduleStatus(task.getScheduleStatus());
    version.setGmtModified(task.getGmtModified());
    String dependencyTaskIds = StringUtils.EMPTY;
    final List<BatchTaskTask> taskTasks = this.batchTaskTaskService.getAllParentTask(task.getId());
    if (CollectionUtils.isNotEmpty(taskTasks)) {
        List<Map<String, Object>> parentTasks = taskTasks.stream().map(taskTask -> {
            Map<String, Object> map = Maps.newHashMap();
            map.put("parentTaskId", taskTask.getParentTaskId());
            return map;
        }).collect(Collectors.toList());
        dependencyTaskIds = JSON.toJSONString(parentTasks);
    }
    version.setDependencyTaskIds(dependencyTaskIds);
    version.setPublishDesc(null == publishDesc ? "" : publishDesc);
    // 插入一条记录信息
    batchTaskVersionService.insert(version);
    task.setVersion(version.getId().intValue());
    return checkVo;
}