Examples with ActionJobEntityVO com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO used on opensource projects

Example 1 with ActionJobEntityVO

use of com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO in project Taier by DTStack.

the class ScheduleActionService method entitys.

/**
 * 根据jobids 和 计算类型，查询job
 */
public List<ActionJobEntityVO> entitys(List<String> jobIds) {
    if (CollectionUtils.isEmpty(jobIds)) {
        throw new RdosDefineException("jobId  is not allow null", ErrorCode.INVALID_PARAMETERS);
    }
    List<ActionJobEntityVO> result = null;
    List<ScheduleJob> scheduleJobs = scheduleJobService.getByJobIds(jobIds);
    if (CollectionUtils.isNotEmpty(scheduleJobs)) {
        result = new ArrayList<>(scheduleJobs.size());
        for (ScheduleJob scheduleJob : scheduleJobs) {
            ActionJobEntityVO vo = new ActionJobEntityVO();
            vo.setJobId(scheduleJob.getJobId());
            vo.setStatus(scheduleJob.getStatus());
            vo.setEngineJobId(scheduleJob.getEngineJobId());
            vo.setApplicationId(scheduleJob.getApplicationId());
            result.add(vo);
        }
    }
    return result;
}

Example 2 with ActionJobEntityVO

use of com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO in project Taier by DTStack.

the class BatchHadoopSelectSqlService method selectRunLog.

/**
 * 获取sql运行日志
 * @param batchTask
 * @param selectSql
 * @param tenantId
 * @param userId
 * @param isRoot
 * @param taskType
 * @return
 * @throws Exception
 */
public ExecuteResultVO selectRunLog(BatchTask batchTask, BatchSelectSql selectSql, Long tenantId, Long userId, Boolean isRoot, Integer taskType) throws Exception {
    String jobId = selectSql.getJobId();
    ExecuteResultVO result = new ExecuteResultVO(jobId);
    // 是否需要脱敏，非admin用户并且是sparkSql needMask才会为true ps:hiveSql不支持脱敏
    if (TempJobType.SIMPLE_SELECT.getType().equals(selectSql.getIsSelectSql())) {
        return result;
    }
    ActionJobEntityVO engineEntity = null;
    // 高级运行的临时表记录和发送到engine的jobId不是同一条记录
    if (StringUtils.isNotEmpty(selectSql.getFatherJobId())) {
        engineEntity = getTaskStatus(selectSql.getFatherJobId());
    } else {
        engineEntity = getTaskStatus(jobId);
    }
    if (engineEntity == null) {
        return result;
    }
    Integer status = TaskStatus.getShowStatus(engineEntity.getStatus());
    result.setStatus(status);
    if (buildLogsWithCheckTaskStatus(selectSql, tenantId, result, StringUtils.isNotEmpty(selectSql.getFatherJobId()) ? selectSql.getFatherJobId() : jobId, engineEntity, status)) {
        return result;
    }
    // update time
    batchSelectSqlService.updateGmtModify(jobId, tenantId);
    return result;
}

Example 3 with ActionJobEntityVO

use of com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO in project Taier by DTStack.

the class BatchHadoopSelectSqlService method getExecuteSqlStatus.

/**
 * 获取sql 执行结果
 *
 * @param selectSql
 * @return
 */
public Integer getExecuteSqlStatus(BatchSelectSql selectSql) {
    Integer status = TaskStatus.UNSUBMIT.getStatus();
    if (selectSql.getIsSelectSql() == TempJobType.SIMPLE_SELECT.getType()) {
        return TaskStatus.FINISHED.getStatus();
    }
    ActionJobEntityVO engineEntity = null;
    // 高级运行的临时表记录和发送到engine的jobId不是同一条记录
    if (StringUtils.isNotEmpty(selectSql.getFatherJobId())) {
        engineEntity = getTaskStatus(selectSql.getFatherJobId());
    } else {
        engineEntity = getTaskStatus(selectSql.getJobId());
    }
    if (Objects.isNull(engineEntity)) {
        return status;
    }
    return TaskStatus.getShowStatus(engineEntity.getStatus());
}

Example 4 with ActionJobEntityVO

use of com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO in project Taier by DTStack.

the class BatchHadoopSelectSqlService method selectData.

/**
 * 从临时表查询数据
 * @param batchTask
 * @param selectSql
 * @param tenantId
 * @param tenantId
 * @param userId
 * @param isRoot
 * @param taskType
 * @return
 * @throws Exception
 */
@Override
public ExecuteResultVO selectData(BatchTask batchTask, BatchSelectSql selectSql, Long tenantId, Long userId, Boolean isRoot, Integer taskType) throws Exception {
    String jobId = selectSql.getJobId();
    ExecuteResultVO result = new ExecuteResultVO(jobId);
    if (selectSql.getIsSelectSql() == TempJobType.SIMPLE_SELECT.getType()) {
        result.setResult(queryData(tenantId, selectSql.getSqlText(), taskType));
        result.setSqlText(selectSql.getSqlText());
    } else {
        ActionJobEntityVO engineEntity = null;
        // 高级运行的临时表记录和发送到engine的jobId不是同一条记录
        if (StringUtils.isNotEmpty(selectSql.getFatherJobId())) {
            engineEntity = getTaskStatus(selectSql.getFatherJobId());
        } else {
            engineEntity = getTaskStatus(jobId);
        }
        if (engineEntity == null) {
            return result;
        }
        Integer status = TaskStatus.getShowStatus(engineEntity.getStatus());
        result.setStatus(status);
        if (buildDataWithCheckTaskStatus(selectSql, tenantId, result, status)) {
            return result;
        }
        // update time
        batchSelectSqlService.updateGmtModify(jobId, tenantId);
    }
    return result;
}

Example 5 with ActionJobEntityVO

use of com.dtstack.taier.scheduler.vo.action.ActionJobEntityVO in project Taier by DTStack.

the class BatchJobService method getSyncTaskStatusInner.

private BatchGetSyncTaskStatusInnerResultVO getSyncTaskStatusInner(final Long tenantId, final String jobId, int retryTimes) {
    final BatchGetSyncTaskStatusInnerResultVO resultVO = new BatchGetSyncTaskStatusInnerResultVO();
    resultVO.setMsg(null);
    resultVO.setStatus(TaskStatus.RUNNING.getStatus());
    try {
        ScheduleJob job = jobService.getScheduleJob(jobId);
        if (Objects.isNull(job)) {
            resultVO.setMsg("无法获取engine数据");
            return resultVO;
        }
        Integer status = TaskStatus.getShowStatus(job.getStatus());
        resultVO.setStatus(status);
        if (TaskStatus.RUNNING.getStatus().equals(status)) {
            resultVO.setMsg("运行中");
        }
        final JSONObject logsBody = new JSONObject(2);
        logsBody.put("jobId", jobId);
        logsBody.put("jobIds", Lists.newArrayList(jobId));
        logsBody.put("computeType", ComputeType.BATCH.getType());
        ActionLogVO actionLogVO = actionService.log(jobId);
        String engineLogStr = actionLogVO.getEngineLog();
        String logInfoStr = actionLogVO.getLogInfo();
        if (StringUtils.isNotBlank(engineLogStr)) {
            // 移除increConf 信息
            try {
                JSONObject engineLogJson = JSON.parseObject(engineLogStr);
                engineLogJson.remove("increConf");
                engineLogStr = engineLogJson.toJSONString();
            } catch (Exception e) {
                LOGGER.error("", e);
                if (TaskStatus.FINISHED.getStatus().equals(status) || TaskStatus.CANCELED.getStatus().equals(status) || TaskStatus.FAILED.getStatus().equals(status)) {
                    resultVO.setMsg(engineLogStr);
                    resultVO.setDownload(String.format(BatchJobService.DOWNLOAD_URL, jobId, EScheduleJobType.SYNC.getVal(), tenantId));
                }
                return resultVO;
            }
        }
        if (StringUtils.isEmpty(engineLogStr) && StringUtils.isEmpty(logInfoStr)) {
            return resultVO;
        }
        try {
            final JSONObject engineLog = JSON.parseObject(engineLogStr);
            final JSONObject logIngo = JSON.parseObject(logInfoStr);
            final StringBuilder logBuild = new StringBuilder();
            // 读取prometheus的相关信息
            Tenant tenantById = this.tenantService.getTenantById(tenantId);
            if (Objects.isNull(tenantById)) {
                LOGGER.info("can not find job tenant{}.", tenantId);
                throw new RdosDefineException(ErrorCode.SERVER_EXCEPTION);
            }
            List<ActionJobEntityVO> engineEntities = actionService.entitys(Collections.singletonList(jobId));
            String engineJobId = "";
            if (CollectionUtils.isNotEmpty(engineEntities)) {
                engineJobId = engineEntities.get(0).getEngineJobId();
            }
            final long startTime = Objects.isNull(job.getExecStartTime()) ? System.currentTimeMillis() : job.getExecStartTime().getTime();
            final String perf = StringUtils.isBlank(engineJobId) ? null : this.batchServerLogService.formatPerfLogInfo(engineJobId, jobId, startTime, System.currentTimeMillis(), tenantById.getId());
            if (StringUtils.isNotBlank(perf)) {
                logBuild.append(perf.replace("\n", "  "));
            }
            if (TaskStatus.FAILED.getStatus().equals(status)) {
                // 失败的话打印失败日志
                logBuild.append("\n");
                logBuild.append("====================Flink日志====================\n");
                if (engineLog != null) {
                    if (StringUtils.isEmpty(engineLog.getString("root-exception")) && retryTimes < 3) {
                        retryTimes++;
                        Thread.sleep(500);
                        return this.getSyncTaskStatusInner(tenantId, jobId, retryTimes);
                    } else {
                        if (engineLog.containsKey("engineLogErr")) {
                            // 有这个字段表示日志没有获取到，目前engine端只对flink任务做了这种处理，这里先提前加上
                            logBuild.append(engineLog.getString("engineLogErr"));
                        } else {
                            logBuild.append(engineLog.getString("root-exception"));
                        }
                        logBuild.append("\n");
                    }
                }
                if (logIngo != null) {
                    logBuild.append(logIngo.getString("msg_info"));
                    logBuild.append("\n");
                }
                final BatchSelectSql batchHiveSelectSql = this.batchSelectSqlService.getByJobId(jobId, tenantId, 0);
                if (batchHiveSelectSql != null) {
                    logBuild.append("====================任务信息====================\n");
                    final String sqlLog = batchHiveSelectSql.getCorrectSqlText().replaceAll("(\"password\"[^\"]+\")([^\"]+)(\")", "$1**$3");
                    logBuild.append(JsonUtils.formatJSON(sqlLog));
                    logBuild.append("\n");
                }
            } else if (TaskStatus.FINISHED.getStatus().equals(status) && retryTimes < 3) {
                // FIXME perjob模式运行任务，任务完成后统计信息可能还没收集到，这里等待1秒后再请求一次结果
                Thread.sleep(1000);
                return this.getSyncTaskStatusInner(tenantId, jobId, 3);
            }
            if (TaskStatus.FINISHED.getStatus().equals(status) || TaskStatus.CANCELED.getStatus().equals(status) || TaskStatus.FAILED.getStatus().equals(status)) {
                resultVO.setDownload(String.format(BatchJobService.DOWNLOAD_URL, jobId, EScheduleJobType.SYNC.getVal(), tenantId));
            }
            resultVO.setMsg(logBuild.toString());
        } catch (Exception e) {
            // 日志解析失败，可能是任务失败，日志信息为非json格式
            LOGGER.error("", e);
            resultVO.setMsg(StringUtils.isEmpty(engineLogStr) ? "engine调度失败" : engineLogStr);
        }
    } catch (Exception e) {
        LOGGER.error("获取同步任务状态失败", e);
    }
    return resultVO;
}