Examples with BatchTaskVersionDetailDTO com.dtstack.taier.dao.dto.BatchTaskVersionDetailDTO used on opensource projects

Example 1 with BatchTaskVersionDetailDTO

use of com.dtstack.taier.dao.dto.BatchTaskVersionDetailDTO in project Taier by DTStack.

the class BatchServerLogService method getLogsByJobId.

public BatchServerLogVO getLogsByJobId(String jobId, Integer pageInfo) {
    if (StringUtils.isBlank(jobId)) {
        return null;
    }
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    final Long tenantId = job.getTenantId();
    final ScheduleTaskShade scheduleTaskShade = this.taskService.findTaskByTaskId(job.getTaskId());
    if (Objects.isNull(scheduleTaskShade)) {
        LOGGER.info("can not find task shade  by jobId:{}.", jobId);
        throw new RdosDefineException(ErrorCode.SERVER_EXCEPTION);
    }
    final BatchServerLogVO batchServerLogVO = new BatchServerLogVO();
    // 日志从engine获取
    final JSONObject logsBody = new JSONObject(2);
    logsBody.put("jobId", jobId);
    logsBody.put("computeType", ComputeType.BATCH.getType());
    ActionLogVO actionLogVO = actionService.log(jobId);
    JSONObject info = new JSONObject();
    if (!Strings.isNullOrEmpty(actionLogVO.getLogInfo())) {
        try {
            info = JSON.parseObject(actionLogVO.getLogInfo());
        } catch (final Exception e) {
            LOGGER.error(String.format("parse jobId： %s  logInfo：%s", jobId, actionLogVO.getLogInfo()), e);
            info.put("msg_info", actionLogVO.getLogInfo());
        }
    }
    if (Objects.nonNull(job.getVersionId())) {
        // 需要获取执行任务时候版本对应的sql
        BatchTaskVersionDetailDTO taskVersion = this.batchTaskVersionService.getByVersionId((long) job.getVersionId());
        if (Objects.nonNull(taskVersion)) {
            if (StringUtils.isEmpty(taskVersion.getOriginSql())) {
                String jsonSql = StringUtils.isEmpty(taskVersion.getSqlText()) ? "{}" : taskVersion.getSqlText();
                scheduleTaskShade.setSqlText(jsonSql);
            } else {
                scheduleTaskShade.setSqlText(taskVersion.getOriginSql());
            }
        }
    }
    info.put("status", job.getStatus());
    if (EScheduleJobType.SPARK_SQL.getVal().equals(scheduleTaskShade.getTaskType())) {
        // 处理sql注释，先把注释base64编码，再处理非注释的自定义参数
        String sql = SqlFormatterUtil.dealAnnotationBefore(scheduleTaskShade.getSqlText());
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        sql = this.jobParamReplace.paramReplace(sql, taskParamsToReplace, job.getCycTime());
        sql = SqlFormatterUtil.dealAnnotationAfter(sql);
        info.put("sql", sql);
    } else if (EScheduleJobType.SYNC.getVal().equals(scheduleTaskShade.getTaskType())) {
        final JSONObject jobJson;
        // taskShade 需要解码
        JSONObject sqlJson = null;
        try {
            sqlJson = JSON.parseObject(Base64Util.baseDecode(scheduleTaskShade.getSqlText()));
        } catch (final Exception e) {
            sqlJson = JSON.parseObject(scheduleTaskShade.getSqlText());
        }
        jobJson = sqlJson.getJSONObject("job");
        // 密码脱敏
        DataFilter.passwordFilter(jobJson);
        String jobStr = jobJson.toJSONString();
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        jobStr = this.jobParamReplace.paramReplace(jobStr, taskParamsToReplace, job.getCycTime());
        info.put("sql", JsonUtils.formatJSON(jobStr));
        if (Objects.nonNull(job.getExecEndTime()) && Objects.nonNull(job.getExecStartTime())) {
            List<ActionJobEntityVO> engineEntities = actionService.entitys(Collections.singletonList(logsBody.getString("jobId")));
            String engineJobId = "";
            if (CollectionUtils.isNotEmpty(engineEntities)) {
                engineJobId = engineEntities.get(0).getEngineJobId();
            }
            this.parseIncreInfo(info, jobStr, tenantId, engineJobId, job.getExecStartTime().getTime(), job.getExecEndTime().getTime(), "");
        }
    }
    if (job.getJobId() != null) {
        try {
            if (StringUtils.isNotBlank(actionLogVO.getEngineLog())) {
                final Map<String, Object> engineLogMap = BatchServerLogService.objectMapper.readValue(actionLogVO.getEngineLog(), Map.class);
                this.dealPerfLog(engineLogMap);
                info.putAll(engineLogMap);
                // 去掉统计信息，界面不展示，调度端统计使用
                info.remove("countInfo");
            }
        } catch (Exception e) {
            // 非json格式的日志也返回
            info.put("msg_info", actionLogVO.getEngineLog());
            LOGGER.error("", e);
        }
    }
    // 增加重试日志
    final String retryLog = this.buildRetryLog(jobId, pageInfo, batchServerLogVO);
    this.formatForLogInfo(info, job.getType(), scheduleTaskShade.getTaskType(), retryLog, null, null, null, batchServerLogVO, tenantId, jobId);
    if (!scheduleTaskShade.getTaskType().equals(EScheduleJobType.SYNC.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.VIRTUAL.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.WORK_FLOW.getVal()) && TaskStatus.getStoppedStatus().contains(job.getStatus())) {
        batchServerLogVO.setDownloadLog(String.format(DOWNLOAD_LOG, jobId, scheduleTaskShade.getTaskType(), 0L));
    }
    batchServerLogVO.setName(scheduleTaskShade.getName());
    batchServerLogVO.setComputeType(scheduleTaskShade.getComputeType());
    batchServerLogVO.setTaskType(scheduleTaskShade.getTaskType());
    return batchServerLogVO;
}

Example 2 with BatchTaskVersionDetailDTO

use of com.dtstack.taier.dao.dto.BatchTaskVersionDetailDTO in project Taier by DTStack.

the class BatchTaskService method getTaskById.

/**
 * 数据开发-根据任务id，查询详情
 *
 * @return
 * @author toutian
 */
public BatchTaskBatchVO getTaskById(final ScheduleTaskVO scheduleTaskVO) {
    final BatchTask task = this.developTaskDao.getOne(scheduleTaskVO.getId());
    if (task == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    final List<BatchResource> resources = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.MAIN_RES.getType());
    final List<BatchResource> refResourceIdList = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.DEPENDENCY_RES.getType());
    final BatchTaskBatchVO taskVO = new BatchTaskBatchVO(this.batchTaskTaskService.getForefathers(task));
    taskVO.setVersion(task.getVersion());
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        // 同步任务类型
        final String taskJson = Base64Util.baseDecode(task.getSqlText());
        if (StringUtils.isBlank(taskJson)) {
            // 向导模式存在为空的情况
            taskVO.setCreateModel(Constant.CREATE_MODEL_GUIDE);
            taskVO.setSqlText("");
        } else {
            final JSONObject obj = JSON.parseObject(taskJson);
            taskVO.setCreateModel(obj.get("createModel") == null ? Constant.CREATE_MODEL_GUIDE : Integer.parseInt(String.valueOf(obj.get("createModel"))));
            formatSqlText(taskVO, obj);
        }
    }
    this.setTaskOperatorModelAndOptions(taskVO, task);
    if (task.getFlowId() != null && task.getFlowId() > 0) {
        taskVO.setFlowId(task.getFlowId());
        final BatchTask flow = this.developTaskDao.getOne(task.getFlowId());
        if (flow != null) {
            taskVO.setFlowName(flow.getName());
        }
    }
    final BatchCatalogue catalogue = batchCatalogueService.getOne(task.getNodePid());
    if (catalogue != null) {
        taskVO.setNodePName(catalogue.getNodeName());
    }
    taskVO.setResourceList(resources);
    taskVO.setRefResourceList(refResourceIdList);
    PageQuery pageQuery = new PageQuery(1, 5, "gmt_create", Sort.DESC.name());
    List<BatchTaskVersionDetailDTO> taskVersions = batchTaskVersionService.listByTaskId(scheduleTaskVO.getId(), pageQuery).stream().map(ver -> {
        if (StringUtils.isNotBlank(ver.getOriginSql())) {
            if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
                ver.setSqlText(ver.getSqlText());
            } else {
                ver.setSqlText(ver.getOriginSql());
            }
        }
        // 填充用户名称
        ver.setUserName(userService.getUserName(ver.getCreateUserId()));
        return ver;
    }).collect(Collectors.toList());
    taskVO.setTaskVersions(taskVersions);
    // 密码脱敏 --2019/10/25 茂茂-- 同步任务 密码脱敏 仅 向导模式 修改成 全部模式
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        try {
            taskVO.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVO.getSqlText())));
        } catch (final Exception e) {
            LOGGER.error("同步任务json解析失败 taskId = {}", task.getId(), e);
            taskVO.setSqlText(DataFilter.passwordFilter(taskVO.getSqlText()));
        }
        for (BatchTaskVersionDetailDTO taskVersion : taskVO.getTaskVersions()) {
            try {
                taskVersion.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVersion.getSqlText())));
            } catch (final Exception e) {
                LOGGER.error("同步任务json解析失败 taskVersionId = {}", taskVersion.getId(), e);
                taskVersion.setSqlText(DataFilter.passwordFilter(taskVersion.getSqlText()));
            }
        }
    }
    final ReadWriteLockVO readWriteLockVO = this.readWriteLockService.getDetail(scheduleTaskVO.getTenantId(), task.getId(), ReadWriteLockType.BATCH_TASK, scheduleTaskVO.getUserId(), task.getModifyUserId(), task.getGmtModified());
    taskVO.setReadWriteLockVO(readWriteLockVO);
    taskVO.setUserId(scheduleTaskVO.getUserId());
    setTaskVariables(taskVO, scheduleTaskVO.getId());
    final List<Long> userIds = new ArrayList<>();
    userIds.add(task.getCreateUserId());
    final Map<Long, User> userMap = userService.getUserMap(userIds);
    buildUserDTOInfo(userMap, taskVO);
    return taskVO;
}

Example 3 with BatchTaskVersionDetailDTO

use of com.dtstack.taier.dao.dto.BatchTaskVersionDetailDTO in project Taier by DTStack.

the class BatchTaskService method getTaskVersionRecord.

public List<BatchTaskVersionDetailDTO> getTaskVersionRecord(Long taskId, Integer pageSize, Integer pageNo) {
    if (pageNo == null) {
        pageNo = 0;
    }
    if (pageSize == null) {
        pageSize = 10;
    }
    PageQuery pageQuery = new PageQuery(pageNo, pageSize, "gmt_create", Sort.DESC.name());
    List<BatchTaskVersionDetailDTO> res = batchTaskVersionService.listByTaskId(taskId, pageQuery);
    for (BatchTaskVersionDetailDTO detail : res) {
        detail.setUserName(userService.getUserName(detail.getCreateUserId()));
    }
    return res;
}

Example 4 with BatchTaskVersionDetailDTO

use of com.dtstack.taier.dao.dto.BatchTaskVersionDetailDTO in project Taier by DTStack.

the class BatchTaskService method taskVersionScheduleConf.

public BatchTaskVersionDetailDTO taskVersionScheduleConf(Long versionId) {
    BatchTaskVersionDetailDTO taskVersion = batchTaskVersionService.getByVersionId(versionId);
    if (taskVersion == null) {
        return null;
    }
    taskVersion.setUserName(userService.getUserName(taskVersion.getCreateUserId()));
    if (StringUtils.isNotBlank(taskVersion.getDependencyTaskIds())) {
        List<Map<String, Object>> dependencyTasks = getDependencyTasks(taskVersion.getDependencyTaskIds());
        JSONObject taskParams = new JSONObject();
        int i = 1;
        for (Map<String, Object> dependencyTask : dependencyTasks) {
            ScheduleTaskShade taskShade = taskService.findTaskByTaskId(MathUtil.getLongVal(dependencyTask.get("parentTaskId")));
            if (taskShade != null) {
                JSONObject taskParam = new JSONObject();
                taskParam.put("taskName", taskShade.getName());
                taskParam.put("tenantName", tenantService.getTenantById(taskShade.getTenantId()).getTenantName());
                taskParams.put("task" + i++, taskParam);
            }
        }
        taskVersion.setDependencyTasks(taskParams);
    }
    return taskVersion;
}