Examples with BatchTask com.dtstack.taier.dao.domain.BatchTask used on opensource projects

Example 21 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method addOrUpdateTask.

/**
 * 数据开发-新建/更新 任务
 *
 * @param param 任务
 * @return
 * @throws NoSuchFieldException
 * @throws IllegalAccessException
 * @author toutian
 */
@Transactional(rollbackFor = Exception.class)
public TaskCatalogueVO addOrUpdateTask(final TaskResourceParam param) {
    // 检查密码回填操作
    this.checkFillPassword(param);
    // 数据预处理 主要是数据同步任务 生成sqlText
    this.checkBeforeUpdateTask(param);
    if (StringUtils.isNotBlank(param.getScheduleConf())) {
        // 处理调度配置
        JSONObject schduleConf = JSON.parseObject(param.getScheduleConf());
        if (schduleConf.get("isExpire") != null && "false".equals(schduleConf.get("isExpire").toString())) {
            schduleConf.replace("isLastInstance", true);
            param.setScheduleConf(schduleConf.toString());
        }
        param.setPeriodType(schduleConf.getInteger("periodType"));
    }
    if (param.getId() > 0 && param.getTaskType().equals(EScheduleJobType.WORK_FLOW.getVal())) {
        // 更新子任务间的依赖关系
        final String sqlText = param.getSqlText();
        if (StringUtils.isNotBlank(sqlText)) {
            final Map<Long, List<Long>> relations = this.parseTaskRelationsFromSqlText(sqlText);
            // 判断任务依赖是否成环
            if (MapUtils.isNotEmpty(relations)) {
                checkIsLoopByList(relations);
            }
            for (final Map.Entry<Long, List<Long>> entry : relations.entrySet()) {
                List<BatchTask> dependencyTasks = getTaskByIds(entry.getValue());
                dependencyTasks.stream().forEach(task -> {
                    task.setTenantId(param.getTenantId());
                });
                batchTaskTaskService.addOrUpdateTaskTask(entry.getKey(), dependencyTasks);
            }
        }
    }
    BatchTaskBatchVO task = PublicUtil.objectToObject(param, BatchTaskBatchVO.class);
    task.setModifyUserId(param.getUserId());
    task.setVersion(Objects.isNull(param.getVersion()) ? 0 : param.getVersion());
    task.parsePeriodType();
    task = this.updateTask(task, param.getEditBaseInfo());
    TaskCatalogueVO taskCatalogueVO = new TaskCatalogueVO(task, task.getNodePid());
    // 强行置为更新
    taskCatalogueVO.getReadWriteLockVO().setResult(TaskLockStatus.TO_UPDATE.getVal());
    // 更新 关联资源
    if (param.getResourceIdList() != null) {
        final Map<String, Object> params = Maps.newHashMap();
        params.put("id", task.getId());
        params.put("resources", param.getResourceIdList());
        params.put("createUserId", task.getCreateUserId());
        this.updateTaskResource(params);
    }
    if (param.getRefResourceIdList() != null) {
        final Map<String, Object> params = Maps.newHashMap();
        params.put("id", task.getId());
        params.put("refResource", param.getRefResourceIdList());
        params.put("createUserId", task.getCreateUserId());
        this.updateTaskRefResource(params);
    }
    final User user = userService.getById(task.getModifyUserId());
    if (user != null) {
        taskCatalogueVO.setCreateUser(user.getUserName());
    }
    final List<BatchTask> dependencyTasks = param.getDependencyTasks();
    if (dependencyTasks != null) {
        this.batchTaskTaskService.addOrUpdateTaskTask(task.getId(), dependencyTasks);
        taskCatalogueVO.setDependencyTasks(dependencyTasks);
    }
    String createUserName = userService.getUserName(task.getCreateUserId());
    taskCatalogueVO.setCreateUser(createUserName);
    taskCatalogueVO.setCatalogueType(CatalogueType.TASK_DEVELOP.getType());
    return taskCatalogueVO;
}

Example 22 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method guideToTemplate.

/**
 * 向导模式转模版
 * @param param
 * @return
 * @throws Exception
 */
@Transactional
public TaskCatalogueVO guideToTemplate(final TaskResourceParam param) {
    final BatchTask task = this.developTaskDao.getOne(param.getId());
    BatchTaskBatchVO taskVO = new BatchTaskBatchVO();
    taskVO.setId(param.getId());
    taskVO.setName(task.getName());
    taskVO.setVersion(param.getVersion());
    taskVO.setUserId(param.getUserId());
    taskVO.setNodePid(task.getNodePid());
    taskVO.setReadWriteLockVO(param.getReadWriteLockVO());
    taskVO.setLockVersion(param.getLockVersion());
    taskVO.setTenantId(param.getTenantId());
    final JSONObject sqlJson = JSON.parseObject(Base64Util.baseDecode(task.getSqlText()));
    sqlJson.put("createModel", TaskCreateModelType.TEMPLATE.getType());
    taskVO.setSqlText(Base64Util.baseEncode(sqlJson.toJSONString()));
    taskVO = this.updateTask(taskVO, true);
    final TaskCatalogueVO taskCatalogueVO = new TaskCatalogueVO(taskVO, taskVO.getNodePid());
    return taskCatalogueVO;
}

Example 23 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchHadoopSelectSqlService method buildSql.

/**
 * 解析sql
 * @param parseResult
 * @param tenantId
 * @param userId
 * @param database
 * @param isCreateAs
 * @param taskId
 * @return
 */
public BuildSqlVO buildSql(ParseResult parseResult, Long tenantId, Long userId, String database, Boolean isCreateAs, Long taskId) {
    BatchTask batchTask = batchTaskService.getBatchTaskById(taskId);
    String originSql = parseResult.getStandardSql();
    // 生成临时表名
    String tempTable = TEMP_TABLE_PREFIX + System.nanoTime();
    if (StringUtils.isEmpty(originSql)) {
        return null;
    }
    String parsedColumns = "";
    int isSelectSql;
    Matcher witchMatcher = WITH_SQL_PATTERN.matcher(originSql);
    String sql = null;
    if (SqlType.CREATE.equals(parseResult.getSqlType())) {
        isSelectSql = TempJobType.CREATE.getType();
        sql = buildCustomFunctionAndDbSql(originSql, tenantId, database, true, batchTask.getTaskType());
    } else if (isCreateAs) {
        isSelectSql = TempJobType.CREATE_AS.getType();
        sql = buildCustomFunctionAndDbSql(originSql, tenantId, database, true, batchTask.getTaskType());
    } else if (SqlType.INSERT.equals(parseResult.getSqlType()) || SqlType.INSERT_OVERWRITE.equals(parseResult.getSqlType())) {
        isSelectSql = TempJobType.INSERT.getType();
        sql = buildCustomFunctionAndDbSql(originSql, tenantId, database, true, batchTask.getTaskType());
    } else if (witchMatcher.find()) {
        TempJobType jobType = getTempJobType(witchMatcher.group("option"));
        isSelectSql = jobType.getType();
        sql = formatSql(jobType, database, tempTable, originSql);
    } else {
        isSelectSql = TempJobType.SELECT.getType();
        sql = buildSelectSqlCustomFunction(originSql, tenantId, database, tempTable, batchTask.getTaskType());
    }
    // 设置需要环境参数
    String taskParam = batchTask.getTaskParams();
    BuildSqlVO buildSqlVO = new BuildSqlVO();
    buildSqlVO.setSql(sql);
    buildSqlVO.setTaskParam(taskParam);
    buildSqlVO.setIsSelectSql(isSelectSql);
    buildSqlVO.setOriginSql(originSql);
    buildSqlVO.setParsedColumns(parsedColumns);
    buildSqlVO.setTenantId(tenantId);
    buildSqlVO.setTempTable(tempTable);
    buildSqlVO.setUserId(userId);
    return buildSqlVO;
}

Example 24 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchSqlExeService method prepareExecuteContent.

/**
 * 进行SQL解析
 * @param executeContent
 */
private void prepareExecuteContent(final ExecuteContent executeContent) {
    BatchTask one = batchTaskService.getOneWithError(executeContent.getTaskId());
    String taskParam = one.getTaskParams();
    ISqlExeService sqlExeService = this.multiEngineServiceFactory.getSqlExeService(executeContent.getTaskType());
    String sql = executeContent.getSql();
    // TODO cache lazy table 暂时不解析血缘，不知道这种类型的sql如何处理
    if (StringUtils.isNotBlank(sql) && (sql.toLowerCase().trim().startsWith("set") || CACHE_LAZY_SQL_PATTEN.matcher(sql).matches())) {
        // set sql 不解析
        ParseResult parseResult = new ParseResult();
        parseResult.setParseSuccess(true);
        parseResult.setOriginSql(executeContent.getSql());
        parseResult.setStandardSql(executeContent.getSql());
        executeContent.setParseResult(parseResult);
        return;
    }
    // 单条sql解析
    if (StringUtils.isNotBlank(executeContent.getSql())) {
        ParseResult parseResult = this.parseSql(executeContent);
        executeContent.setParseResult(parseResult);
        // 校验语法
        if (executeContent.isCheckSyntax()) {
            sqlExeService.checkSingleSqlSyntax(executeContent.getTenantId(), executeContent.getSql(), executeContent.getDatabase(), taskParam);
        }
    }
    // 批量解析sql
    List<ParseResult> parseResultList = Lists.newLinkedList();
    if (CollectionUtils.isNotEmpty(executeContent.getSqlList())) {
        String finalTaskParam = taskParam;
        executeContent.getSqlList().forEach(x -> {
            if (!x.trim().startsWith("set")) {
                if (executeContent.isCheckSyntax()) {
                    executeContent.setSql(x);
                    ParseResult batchParseResult = this.parseSql(executeContent);
                    sqlExeService.checkSingleSqlSyntax(executeContent.getTenantId(), x, executeContent.getDatabase(), finalTaskParam);
                    parseResultList.add(batchParseResult);
                }
            } else {
                // set sql 不解析
                ParseResult batchParseResult = new ParseResult();
                batchParseResult.setParseSuccess(true);
                batchParseResult.setOriginSql(x);
                batchParseResult.setStandardSql(x);
                parseResultList.add(batchParseResult);
            }
        });
        executeContent.setParseResultList(parseResultList);
    }
}

Example 25 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskTaskService method getTaskTasksByTaskIdAndTasks.

/**
 * 根据taskId和List<BatchTask>生成List<BatchTaskTask>
 * @param taskId
 * @param tasks
 * @return
 */
private List<BatchTaskTask> getTaskTasksByTaskIdAndTasks(Long taskId, List<BatchTask> tasks) {
    List<BatchTaskTask> taskTasks = Lists.newArrayList();
    for (BatchTask task : tasks) {
        BatchTaskTask taskTask = new BatchTaskTask();
        taskTask.setParentTaskId(task.getId());
        taskTask.setTenantId(task.getTenantId());
        taskTask.setTaskId(taskId);
        taskTasks.add(taskTask);
    }
    return taskTasks;
}