Examples with BatchTaskParamShade com.dtstack.taier.dao.domain.BatchTaskParamShade used on opensource projects

Example 1 with BatchTaskParamShade

use of com.dtstack.taier.dao.domain.BatchTaskParamShade in project Taier by DTStack.

the class BatchServerLogService method getLogsByJobId.

public BatchServerLogVO getLogsByJobId(String jobId, Integer pageInfo) {
    if (StringUtils.isBlank(jobId)) {
        return null;
    }
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    final Long tenantId = job.getTenantId();
    final ScheduleTaskShade scheduleTaskShade = this.taskService.findTaskByTaskId(job.getTaskId());
    if (Objects.isNull(scheduleTaskShade)) {
        LOGGER.info("can not find task shade  by jobId:{}.", jobId);
        throw new RdosDefineException(ErrorCode.SERVER_EXCEPTION);
    }
    final BatchServerLogVO batchServerLogVO = new BatchServerLogVO();
    // 日志从engine获取
    final JSONObject logsBody = new JSONObject(2);
    logsBody.put("jobId", jobId);
    logsBody.put("computeType", ComputeType.BATCH.getType());
    ActionLogVO actionLogVO = actionService.log(jobId);
    JSONObject info = new JSONObject();
    if (!Strings.isNullOrEmpty(actionLogVO.getLogInfo())) {
        try {
            info = JSON.parseObject(actionLogVO.getLogInfo());
        } catch (final Exception e) {
            LOGGER.error(String.format("parse jobId： %s  logInfo：%s", jobId, actionLogVO.getLogInfo()), e);
            info.put("msg_info", actionLogVO.getLogInfo());
        }
    }
    if (Objects.nonNull(job.getVersionId())) {
        // 需要获取执行任务时候版本对应的sql
        BatchTaskVersionDetailDTO taskVersion = this.batchTaskVersionService.getByVersionId((long) job.getVersionId());
        if (Objects.nonNull(taskVersion)) {
            if (StringUtils.isEmpty(taskVersion.getOriginSql())) {
                String jsonSql = StringUtils.isEmpty(taskVersion.getSqlText()) ? "{}" : taskVersion.getSqlText();
                scheduleTaskShade.setSqlText(jsonSql);
            } else {
                scheduleTaskShade.setSqlText(taskVersion.getOriginSql());
            }
        }
    }
    info.put("status", job.getStatus());
    if (EScheduleJobType.SPARK_SQL.getVal().equals(scheduleTaskShade.getTaskType())) {
        // 处理sql注释，先把注释base64编码，再处理非注释的自定义参数
        String sql = SqlFormatterUtil.dealAnnotationBefore(scheduleTaskShade.getSqlText());
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        sql = this.jobParamReplace.paramReplace(sql, taskParamsToReplace, job.getCycTime());
        sql = SqlFormatterUtil.dealAnnotationAfter(sql);
        info.put("sql", sql);
    } else if (EScheduleJobType.SYNC.getVal().equals(scheduleTaskShade.getTaskType())) {
        final JSONObject jobJson;
        // taskShade 需要解码
        JSONObject sqlJson = null;
        try {
            sqlJson = JSON.parseObject(Base64Util.baseDecode(scheduleTaskShade.getSqlText()));
        } catch (final Exception e) {
            sqlJson = JSON.parseObject(scheduleTaskShade.getSqlText());
        }
        jobJson = sqlJson.getJSONObject("job");
        // 密码脱敏
        DataFilter.passwordFilter(jobJson);
        String jobStr = jobJson.toJSONString();
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        jobStr = this.jobParamReplace.paramReplace(jobStr, taskParamsToReplace, job.getCycTime());
        info.put("sql", JsonUtils.formatJSON(jobStr));
        if (Objects.nonNull(job.getExecEndTime()) && Objects.nonNull(job.getExecStartTime())) {
            List<ActionJobEntityVO> engineEntities = actionService.entitys(Collections.singletonList(logsBody.getString("jobId")));
            String engineJobId = "";
            if (CollectionUtils.isNotEmpty(engineEntities)) {
                engineJobId = engineEntities.get(0).getEngineJobId();
            }
            this.parseIncreInfo(info, jobStr, tenantId, engineJobId, job.getExecStartTime().getTime(), job.getExecEndTime().getTime(), "");
        }
    }
    if (job.getJobId() != null) {
        try {
            if (StringUtils.isNotBlank(actionLogVO.getEngineLog())) {
                final Map<String, Object> engineLogMap = BatchServerLogService.objectMapper.readValue(actionLogVO.getEngineLog(), Map.class);
                this.dealPerfLog(engineLogMap);
                info.putAll(engineLogMap);
                // 去掉统计信息，界面不展示，调度端统计使用
                info.remove("countInfo");
            }
        } catch (Exception e) {
            // 非json格式的日志也返回
            info.put("msg_info", actionLogVO.getEngineLog());
            LOGGER.error("", e);
        }
    }
    // 增加重试日志
    final String retryLog = this.buildRetryLog(jobId, pageInfo, batchServerLogVO);
    this.formatForLogInfo(info, job.getType(), scheduleTaskShade.getTaskType(), retryLog, null, null, null, batchServerLogVO, tenantId, jobId);
    if (!scheduleTaskShade.getTaskType().equals(EScheduleJobType.SYNC.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.VIRTUAL.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.WORK_FLOW.getVal()) && TaskStatus.getStoppedStatus().contains(job.getStatus())) {
        batchServerLogVO.setDownloadLog(String.format(DOWNLOAD_LOG, jobId, scheduleTaskShade.getTaskType(), 0L));
    }
    batchServerLogVO.setName(scheduleTaskShade.getName());
    batchServerLogVO.setComputeType(scheduleTaskShade.getComputeType());
    batchServerLogVO.setTaskType(scheduleTaskShade.getTaskType());
    return batchServerLogVO;
}

Example 2 with BatchTaskParamShade

use of com.dtstack.taier.dao.domain.BatchTaskParamShade in project Taier by DTStack.

the class BatchTaskParamShadeService method getTaskParam.

public List<BatchTaskParamShade> getTaskParam(long taskId) {
    List<BatchTaskParamShade> taskParamShades = developTaskParamShadeDao.listByTaskId(taskId);
    // 特殊处理 TaskParam 系统参数
    for (BatchTaskParamShade taskParamShade : taskParamShades) {
        if (EParamType.SYS_TYPE.getType() != taskParamShade.getType()) {
            continue;
        }
        // 将 command 属性设置为系统表的 command
        BatchSysParameter sysParameter = batchSysParamService.getBatchSysParamByName(taskParamShade.getParamName());
        taskParamShade.setParamCommand(sysParameter.getParamCommand());
    }
    return taskParamShades;
}

Example 3 with BatchTaskParamShade

use of com.dtstack.taier.dao.domain.BatchTaskParamShade in project Taier by DTStack.

the class BatchTaskParamShadeService method saveTaskParam.

public void saveTaskParam(List<BatchTaskParam> paramList) {
    for (BatchTaskParam batchTaskParam : paramList) {
        BatchTaskParamShade paramShade = new BatchTaskParamShade();
        BeanUtils.copyProperties(batchTaskParam, paramShade);
        addOrUpdate(paramShade);
    }
}

Example 4 with BatchTaskParamShade

use of com.dtstack.taier.dao.domain.BatchTaskParamShade in project Taier by DTStack.

the class BatchTaskParamShadeService method addOrUpdate.

public void addOrUpdate(BatchTaskParamShade batchTaskParamShade) {
    if (StringUtils.isBlank(batchTaskParamShade.getParamCommand())) {
        throw new RdosDefineException("自定义参数赋值不能为空");
    }
    BatchTaskParamShade dbTaskParam = developTaskParamShadeDao.getByTypeAndName(batchTaskParamShade.getTaskId(), batchTaskParamShade.getType(), batchTaskParamShade.getParamName());
    if (Objects.nonNull(dbTaskParam)) {
        dbTaskParam.setParamCommand(batchTaskParamShade.getParamCommand());
        dbTaskParam.setGmtModified(new Timestamp(System.currentTimeMillis()));
        developTaskParamShadeDao.update(dbTaskParam);
    } else {
        developTaskParamShadeDao.insert(batchTaskParamShade);
    }
}

Example 5 with BatchTaskParamShade

use of com.dtstack.taier.dao.domain.BatchTaskParamShade in project Taier by DTStack.

the class JobParamReplace method paramReplace.

public String paramReplace(String sql, List paramList, String cycTime) {
    if (CollectionUtils.isEmpty(paramList)) {
        return sql;
    }
    Matcher matcher = PARAM_PATTERN.matcher(sql);
    if (!matcher.find()) {
        return sql;
    }
    for (Object param : paramList) {
        Integer type;
        String paramName;
        String paramCommand;
        if (param instanceof BatchTaskParamShade) {
            type = ((BatchTaskParamShade) param).getType();
            paramName = ((BatchTaskParamShade) param).getParamName();
            paramCommand = ((BatchTaskParamShade) param).getParamCommand();
        } else {
            type = ((BatchTaskParam) param).getType();
            paramName = ((BatchTaskParam) param).getParamName();
            paramCommand = ((BatchTaskParam) param).getParamCommand();
        }
        String replaceStr = String.format(VAR_FORMAT, paramName);
        // 判断参数是否存在 SQL 中
        if (!sql.contains(replaceStr)) {
            continue;
        }
        String targetVal = convertParam(type, paramName, paramCommand, cycTime);
        sql = sql.replace(replaceStr, targetVal);
    }
    return sql;
}