Examples with UnExpectedRequestException com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException used on opensource projects

Example 21 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method killApplication.

@Override
public GeneralResponse<?> killApplication(Application applicationInDb, String user) throws JobKillException, UnExpectedRequestException, ClusterInfoNotConfigException {
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    List<JobKillResult> results = new ArrayList<>();
    if (tasks == null || tasks.isEmpty()) {
        throw new UnExpectedRequestException("Sub tasks {&CAN_NOT_BE_NULL_OR_EMPTY}");
    }
    for (Task task : tasks) {
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(task.getClusterName());
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException("Failed to find cluster id: " + task.getClusterName() + " configuration");
        }
        results.add(abstractJobSubmitter.killJob(user, clusterInfo.getClusterName(), task));
        task.setStatus(TaskStatusEnum.CANCELLED.getCode());
        task.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
        taskDao.save(task);
    }
    return new GeneralResponse<>("200", "{&SUCCESS_TO_KILL_TASK}", results.size());
}

Example 22 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method executeFileRule.

@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    LOGGER.info("Start to execute file rule task and save check result.");
    Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
    Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
    int totalRules = fileRules.size();
    int successRule = 0;
    for (Rule rule : fileRules) {
        if (rule.getAbortOnFailure() != null) {
            taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
        }
        TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
        taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
        RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
        taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
        // Check rule datasource: 1) table 2) partition.
        if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
            TableStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getTableSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        } else {
            PartitionStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getPartitionSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        }
        if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            break;
        }
    }
    taskInDb.setTaskDataSources(taskDataSources);
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Update task status
    taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    if (totalRules == successRule) {
        taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    taskDao.save(taskInDb);
    LOGGER.info("Finished to execute file rule task and save check result.");
    TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
    taskSubmitResult.setApplicationId(application.getId());
    taskSubmitResult.setClusterName(clusterName);
    return taskSubmitResult;
}

Example 23 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method saveTaskRusult.

private List<TaskResult> saveTaskRusult(String fullSize, double fileCount, Application application, String submitTime, Rule rule, Set<AlarmConfig> alarmConfig, String runDate) throws UnExpectedRequestException {
    double number = Double.parseDouble(fullSize.split(" ")[0]);
    String unit = fullSize.split(" ")[1];
    // Save task result.
    List<TaskResult> taskResults = new ArrayList<>();
    List<Integer> fileOutputNames = alarmConfig.stream().map(AlarmConfig::getFileOutputName).distinct().collect(Collectors.toList());
    for (Integer fileOutputName : fileOutputNames) {
        AlarmConfig currentAlarmConfig = alarmConfig.stream().filter(alarmConfigSetting -> alarmConfigSetting.getFileOutputName().equals(fileOutputName)).iterator().next();
        RuleMetric ruleMetric = currentAlarmConfig.getRuleMetric();
        if (ruleMetric == null) {
            throw new UnExpectedRequestException("File rule metric {&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        TaskResult taskResult;
        if (StringUtils.isNotBlank(runDate)) {
            TaskResult existTaskResult = taskResultDao.find(runDate, rule.getId(), ruleMetric.getId());
            if (existTaskResult != null) {
                taskResult = existTaskResult;
            } else {
                taskResult = new TaskResult();
                Date runRealDate = null;
                try {
                    runRealDate = new SimpleDateFormat("yyyyMMdd").parse(runDate);
                } catch (ParseException e) {
                    String errorMsg = "Parse date string with run date failed. Exception message: " + e.getMessage();
                    LOGGER.error(errorMsg);
                    throw new UnExpectedRequestException(errorMsg);
                }
                taskResult.setRunDate(runRealDate.getTime());
            }
        } else {
            taskResult = new TaskResult();
        }
        taskResult.setApplicationId(application.getId());
        taskResult.setCreateTime(submitTime);
        taskResult.setRuleId(rule.getId());
        taskResult.setRuleMetricId(ruleMetric.getId());
        if (fileOutputName.equals(FileOutputNameEnum.FILE_COUNT.getCode())) {
            taskResult.setResultType("int");
            taskResult.setValue(fileCount + "");
        } else if (fileOutputName.equals(FileOutputNameEnum.DIR_SIZE.getCode())) {
            String alarmConfigUnit = FileOutputUnitEnum.fileOutputUnit(currentAlarmConfig.getFileOutputUnit());
            taskResult.setResultType(alarmConfigUnit);
            taskResult.setValue(UnitTransfer.alarmconfigToTaskResult(number, alarmConfigUnit, unit.toUpperCase()) + "");
        } else {
            throw new UnExpectedRequestException("Unknown file output name.");
        }
        taskResults.add(taskResultDao.saveTaskResult(taskResult));
    }
    return taskResults;
}

Example 24 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class LinkisConfiguration method saveFullTree.

public Map saveFullTree(String clusterName, String creator, List<Map> fullTreeQueueName, List<Map> fullTree, String userName) throws UnExpectedRequestException {
    ClusterInfo clusterInfoInDb = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfoInDb == null) {
        throw new UnExpectedRequestException("cluster name {&ALREADY_EXIST}");
    }
    String url = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getSaveFullTree()).toString();
    Gson gson = new Gson();
    Map<String, Object> map = new HashMap<>(2);
    map.put("creator", creator);
    map.put("fullTree", fullTree);
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", userName);
    headers.add("Token-Code", clusterInfoInDb.getLinkisToken());
    HttpEntity entity = new HttpEntity<>(gson.toJson(map), headers);
    Map response = null;
    Map responseQueueName = null;
    LOGGER.info("Start to save configuration to linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity);
    try {
        response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class).getBody();
        LOGGER.info("Finish to save configuration to linkis. response: {}", response);
        Integer code = (Integer) response.get("status");
        if (code != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        LOGGER.info("Failed to get configuration from linkis.");
        throw new UnExpectedRequestException("Failed to get configuration from linkis.");
    }
    return (Map) response.get("data");
}

Example 25 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class LinkisConfiguration method getFullTree.

public Map getFullTree(String clusterName, String user) throws UnExpectedRequestException {
    ClusterInfo clusterInfoInDb = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfoInDb == null) {
        throw new UnExpectedRequestException("cluster name {&DOES_NOT_EXIST}");
    }
    String url = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getGetFullTree()).queryParam("creator", linkisConfig.getAppName()).queryParam("engineType", "spark").queryParam("version", "2.4.3").toString();
    String urlQueueName = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getGetFullTree()).queryParam("creator", linkisConfig.getAppName()).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", user);
    headers.add("Token-Code", clusterInfoInDb.getLinkisToken());
    HttpEntity entity = new HttpEntity<>(headers);
    HttpEntity entityQueueName = new HttpEntity<>(headers);
    Map response = null;
    Map responseQueueName = null;
    LOGGER.info("Start to get configuration from linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    try {
        response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
        responseQueueName = restTemplate.exchange(urlQueueName, HttpMethod.GET, entityQueueName, Map.class).getBody();
        LOGGER.info("Finish to get configuration from linkis. response: {}", response);
        Integer code = (Integer) response.get("status");
        if (code != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
        Integer codeQueueName = (Integer) responseQueueName.get("status");
        if (codeQueueName != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        LOGGER.info("Failed to get configuration from linkis.");
    }
    Map<String, Map> responseMap = new HashMap<>(2);
    responseMap.put("fule_tree", (Map) response.get("data"));
    responseMap.put("full_tree_queue_name", (Map) responseQueueName.get("data"));
    return responseMap;
}