Examples with Task com.webank.wedatasphere.qualitis.entity.Task used on opensource projects

Example 6 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method getTaskLog.

@Override
public GeneralResponse<?> getTaskLog(GetTaskLogRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Check Arguments
    GetTaskLogRequest.checkRequest(request);
    Task task = taskDao.findByRemoteTaskIdAndClusterName(request.getTaskId(), request.getClusterId());
    if (task == null) {
        throw new UnExpectedRequestException("Task_id {&DOES_NOT_EXIST}");
    }
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(request.getClusterId());
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("cluster : [" + request.getClusterId() + "] {&DOES_NOT_EXIST}");
    }
    String executeUser = task.getApplication().getExecuteUser();
    // Check if user has permissions proxying execution user
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), executeUser);
    // Check if user has permissions to view this task
    if (!request.getCreateUser().equals(task.getApplication().getCreateUser())) {
        throw new UnExpectedRequestException(String.format("User: %s {&HAS_NO_PERMISSION_TO_ACCESS} task: %s", request.getCreateUser(), request.getTaskId()), 403);
    }
    LogResult logResult;
    try {
        logResult = monitorManager.getTaskPartialLog(request.getTaskId(), 0, executeUser, clusterInfo.getLinkisAddress(), clusterInfo.getClusterName());
    } catch (LogPartialException | ClusterInfoNotConfigException e) {
        throw new UnExpectedRequestException(e.getMessage());
    }
    LOGGER.info("Succeed to get log of the task. task_id: {}, cluster_id: {}", request.getTaskId(), request.getClusterId());
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_LOG}", logResult.getLog());
}

Example 7 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method getApplicationResult.

@Override
public GeneralResponse<?> getApplicationResult(String applicationId) throws UnExpectedRequestException {
    // Find application by applicationId
    Application applicationInDb = applicationDao.findById(applicationId);
    if (applicationInDb == null) {
        throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_EXIST}", applicationId));
    }
    Integer passNum = 0;
    Integer failedNum = 0;
    Integer notPassNum = 0;
    StringBuilder passMessage = new StringBuilder();
    StringBuilder failedMessage = new StringBuilder();
    StringBuilder notPassMessage = new StringBuilder();
    passMessage.append("The rules following below are pass(以下规则已通过校验): [");
    failedMessage.append("The rules following below are failed(以下规则已失败): [");
    notPassMessage.append("The rules following below are failed(以下规则未通过校验): [");
    // Find all tasks by application
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    // Find task rule simple in all tasks
    for (Task task : tasks) {
        // Add failed num, pass num if task status equals to Succeed, Failed
        if (task.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            failedNum += task.getTaskRuleSimples().size();
            generateFailedMessage(task, failedMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.PASS_CHECKOUT.getCode())) {
            passNum += task.getTaskRuleSimples().size();
            generatePassMessage(task, passMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.FAIL_CHECKOUT.getCode())) {
            // Find not pass task if task status equals to failed_checkout
            for (TaskRuleSimple taskRuleSimple : task.getTaskRuleSimples()) {
                Boolean isPass = true;
                for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
                    if (taskRuleAlarmConfig.getStatus().equals(AlarmConfigStatusEnum.NOT_PASS.getCode())) {
                        isPass = false;
                        break;
                    }
                }
                if (isPass) {
                    passNum++;
                    passMessage.append(taskRuleSimple.getRuleName()).append(", ");
                } else {
                    notPassNum++;
                    notPassMessage.append(taskRuleSimple.getRuleName()).append(", ");
                }
            }
        } else {
            throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_FINISHED_YET}", applicationId));
        }
    }
    passMessage.append("]").append(System.lineSeparator());
    failedMessage.append("]").append(System.lineSeparator());
    notPassMessage.append("]").append(System.lineSeparator());
    String resultMessage = passMessage.toString() + failedMessage.toString() + notPassMessage.toString();
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_RESULT}", new ApplicationResultResponse(passNum, failedNum, notPassNum, resultMessage));
}

Example 8 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class TaskChecker method checkTaskStatus.

@Override
@Transactional(rollbackFor = Exception.class)
public void checkTaskStatus(JobChecker jobChecker) {
    try {
        Map<String, Object> taskInfos = monitorManager.getTaskStatus(jobChecker.getTaskId(), jobChecker.getUsername(), jobChecker.getUjesAddress(), jobChecker.getClusterName());
        String jobStatus = ((String) taskInfos.get("status")).toUpperCase();
        Integer errCode = (Integer) taskInfos.get("errCode");
        LOGGER.info("Task status: {}", jobStatus);
        if (!jobStatus.equals(jobChecker.getOldStatus())) {
            LOGGER.info("Start to update task status. old status: {}, new status: {}, task_id: {}", jobChecker.getOldStatus(), jobStatus, jobChecker.getTaskId());
            writeDb(jobChecker, jobStatus, errCode);
            LOGGER.info("Succeed to update task status. old status: {}, new status: {}, task_id: {}", jobChecker.getOldStatus(), jobStatus, jobChecker.getTaskId());
        }
        // Compute task time in same progress.
        if (linkisConfig.getKillStuckTasks() && TaskStatusEnum.RUNNING.getState().equals(jobStatus)) {
            Task taskInDb = taskDao.findByRemoteTaskIdAndClusterName(jobChecker.getTaskId(), jobChecker.getClusterName());
            Double progress = (Double) taskInfos.get("progress");
            LOGGER.info("Old time progress[{}].", jobChecker.getOldProgress());
            LOGGER.info("Current time progress[{}].", progress);
            long runningTime = System.currentTimeMillis() - taskInDb.getRunningTime();
            LOGGER.info("Current task running time [{}] minutes.", runningTime / (60 * 1000));
            if (progress.equals(jobChecker.getOldProgress())) {
                long diff = System.currentTimeMillis() - taskInDb.getNewProgressTime();
                long diffMinutes = diff;
                LOGGER.info("Time in same progress[{}]: {} minutes. Config max time: {} minutes.", progress, diffMinutes / (60 * 1000), linkisConfig.getKillStuckTasksTime().longValue() / (60 * 1000));
                if (diffMinutes > linkisConfig.getKillStuckTasksTime().longValue()) {
                    killTimeoutTask(applicationDao.findById(jobChecker.getApplicationId()), taskInDb, jobChecker);
                }
            } else {
                LOGGER.info("Progress is updating , so is task new progress.");
                taskInDb.setNewProgressTime(System.currentTimeMillis());
                taskInDb.setProgress(progress);
                if (runningTime > linkisConfig.getKillStuckTasksTime().longValue()) {
                    killTimeoutTask(applicationDao.findById(jobChecker.getApplicationId()), taskInDb, jobChecker);
                }
            }
            taskDao.save(taskInDb);
        }
    } catch (TaskNotExistException e) {
        LOGGER.error("Spark Task [{}] does not exist, application id : [{}]", jobChecker.getTaskId(), jobChecker.getApplicationId(), e);
        jobChecker.getTask().setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
        taskDao.save(jobChecker.getTask());
        jobChecker.getTask().getApplication().addAbnormalTaskNum();
        applicationDao.saveApplication(jobChecker.getTask().getApplication());
    } catch (Exception e) {
        LOGGER.error("Check task id:[{}] failed, application id:[{}]", jobChecker.getTaskId(), jobChecker.getApplicationId(), e);
    }
}

Example 9 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class CheckerRunnable method getJobs.

private List<JobChecker> getJobs() {
    List<Application> notEndApplications = applicationDao.findByStatusNotIn(END_APPLICATION_STATUS_LIST);
    List<JobChecker> jobCheckers = new ArrayList<>();
    for (Application app : notEndApplications) {
        // Find not end task
        List<Task> notEndTasks = taskDao.findByApplicationAndStatusInAndTaskRemoteIdNotNull(app, NOT_END_TASK_STATUS_LIST);
        for (Task task : notEndTasks) {
            JobChecker tmp = new JobChecker(app.getId(), TaskStatusEnum.getTaskStateByCode(task.getStatus()), task.getProgress(), StringUtils.isNotBlank(task.getTaskProxyUser()) ? task.getTaskProxyUser() : app.getExecuteUser(), task.getSubmitAddress(), task.getClusterName(), task);
            jobCheckers.add(tmp);
        }
        if (notEndTasks.isEmpty()) {
            LOGGER.info("Find abnormal application, which tasks is all end, but application is not end.");
            List<Task> allTasks = taskDao.findByApplication(app);
            app.resetTask();
            applicationDao.saveApplication(app);
            LOGGER.info("Finish to reset application status num.");
            LOGGER.info("Start to recover application status.");
            try {
                for (Task task : allTasks) {
                    if (task.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
                        iChecker.checkIfLastJob(app, false, false, false);
                    } else if (task.getAbortOnFailure() != null && !task.getAbortOnFailure() && task.getStatus().equals(TaskStatusEnum.FAIL_CHECKOUT.getCode())) {
                        iChecker.checkIfLastJob(app, true, false, false);
                    } else if (task.getStatus().equals(TaskStatusEnum.PASS_CHECKOUT.getCode())) {
                        iChecker.checkIfLastJob(app, true, true, false);
                    } else if (task.getStatus().equals(TaskStatusEnum.TASK_NOT_EXIST.getCode())) {
                        iChecker.checkIfLastJob(app, false, false, true);
                    } else if (task.getStatus().equals(TaskStatusEnum.CANCELLED.getCode())) {
                        app.setApplicationComment(ApplicationCommentEnum.TIMEOUT_KILL.getCode());
                        iChecker.checkIfLastJob(app, false, false, false);
                    }
                }
                LOGGER.info("Succeed to recover application status.");
            } catch (Exception e) {
                LOGGER.error("Failed to recover applications that are not end.");
                LOGGER.error(e.getMessage(), e);
            }
        }
    }
    return jobCheckers;
}

Example 10 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterStatusApplication.

@Override
public GeneralResponse<?> filterStatusApplication(FilterStatusRequest request) throws UnExpectedRequestException {
    // Check arguments
    FilterStatusRequest.checkRequest(request);
    String userName = HttpUtils.getUserName(httpServletRequest);
    if (request.getStatus() != null) {
        LOGGER.info("User: {} wants to find applications with status: {}", userName, request.getStatus());
    } else {
        LOGGER.info("User: {} wants to find all applications", userName);
    }
    List<Application> applicationList;
    Long total;
    Integer page = request.getPage();
    Integer size = request.getSize();
    if (request.getStatus() == null || request.getStatus().intValue() == 0) {
        // Paging find applications by user
        long currentTimeUser = System.currentTimeMillis();
        applicationList = applicationDao.findByCreateUser(userName, page, size);
        LOGGER.info("timechecker find page application :" + (System.currentTimeMillis() - currentTimeUser));
        long currentTimeCountUser = System.currentTimeMillis();
        total = applicationDao.countByCreateUser(userName);
        LOGGER.info("timechecker count application :" + (System.currentTimeMillis() - currentTimeCountUser));
    } else {
        // Paging find applications by user and status
        applicationList = applicationDao.findByCreateUserAndStatus(userName, request.getStatus(), request.getCommentType(), page, size);
        total = applicationDao.countByCreateUserAndStatus(userName, request.getStatus(), request.getCommentType());
    }
    long currentTimeResponse = System.currentTimeMillis();
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(userName) || application.getExecuteUser().equals(userName)) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("timechecker response :" + (System.currentTimeMillis() - currentTimeResponse));
    LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}