Examples with UnExpectedRequestException com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException used on opensource projects

Example 71 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method writeExcelFile.

private void writeExcelFile(File tmpFile, List<String> tables, UploadResultRequest request) throws UnExpectedRequestException {
    int sheetNo = 1;
    OutputStream tmpOutputStream = null;
    try {
        Path parentPath = Paths.get(tmpFile.getParent());
        if (Files.notExists(parentPath)) {
            tmpFile.getParentFile().mkdirs();
        }
        if (!tmpFile.exists()) {
            LOGGER.info("Start to create local file.");
            tmpFile.createNewFile();
        }
        tmpOutputStream = new FileOutputStream(tmpFile);
        ExcelWriter writer = new ExcelWriter(tmpOutputStream, ExcelTypeEnum.XLSX, true);
        // Find task with the start time and end time.
        for (String tableName : tables) {
            List<Task> tasks = taskDao.findWithSubmitTimeAndDatasource(request.getStartTime(), request.getEndTime(), request.getClusterName(), request.getDatabaseName(), tableName);
            List<TaskRuleSimple> taskRuleSimples = tasks.stream().map(Task::getTaskRuleSimples).flatMap(taskRuleSimpleSet -> taskRuleSimpleSet.stream()).distinct().collect(Collectors.toList());
            // Generate analysis result excel
            List<ExcelResult> results = new ArrayList<>(taskRuleSimples.size());
            for (TaskRuleSimple taskRuleSimple : taskRuleSimples) {
                ExcelResult excelResult = new ExcelResult();
                excelResult.setRuleName(taskRuleSimple.getRuleName());
                excelResult.setClusterName(request.getClusterName());
                excelResult.setDatabaseName(request.getDatabaseName());
                excelResult.setTableName(tableName);
                excelResult.setBeginTime(taskRuleSimple.getTask().getBeginTime());
                excelResult.setEndTime(taskRuleSimple.getTask().getEndTime());
                StringBuffer checkTemplateStr = new StringBuffer();
                StringBuffer resultStr = new StringBuffer();
                joinAlarmConfig(results, taskRuleSimple, excelResult, request, checkTemplateStr, resultStr);
                results.add(excelResult);
            }
            Sheet templateSheet = new Sheet(sheetNo++, 0, ExcelResult.class);
            templateSheet.setSheetName(tableName + "-" + ExcelSheetName.ANALYSIS_NAME);
            writer.write(results, templateSheet);
        }
        writer.finish();
    } catch (FileNotFoundException e) {
        LOGGER.error(e.getMessage(), e);
        throw new UnExpectedRequestException("{&FAILED_TO_CREATE_LOCAL_FILE}");
    } catch (IOException e) {
        LOGGER.error(e.getMessage(), e);
        throw new UnExpectedRequestException("{&FAILED_TO_CREATE_LOCAL_FILE}");
    } finally {
        try {
            tmpOutputStream.close();
        } catch (IOException e) {
            LOGGER.error(e.getMessage(), e);
        }
    }
}

Example 72 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method uploadDataSourceAnalysisResult.

@Override
public GeneralResponse<?> uploadDataSourceAnalysisResult(UploadResultRequest request) throws UnExpectedRequestException, IOException {
    // Login user permission.
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    if (user == null) {
        throw new UnExpectedRequestException("User {&DOES_NOT_EXIST}");
    }
    UploadResultRequest.checkRequest(request);
    List<ApplicationClusterResponse> responses = (List<ApplicationClusterResponse>) getDataSource(new PageRequest(0, Integer.MAX_VALUE)).getData();
    List<String> tables = responses.stream().filter(cluster -> cluster.getClusterName().equals(request.getClusterName())).map(ApplicationClusterResponse::getDatabase).flatMap(database -> database.stream()).filter(databaseResponse -> databaseResponse.getDatabaseName().equals(request.getDatabaseName())).map(ApplicationDatabaseResponse::getTable).flatMap(table -> table.stream()).distinct().collect(Collectors.toList());
    if (StringUtils.isNotBlank(request.getTableName())) {
        tables.clear();
        tables.add(request.getTableName());
    }
    LOGGER.info("Start to write excel");
    StringBuffer fileName = new StringBuffer();
    fileName.append(linkisConfig.getUploadTmpPath()).append(File.separator).append(user.getUserName()).append("_").append(request.getClusterName()).append("_").append(request.getDatabaseName()).append("_").append(UUID.randomUUID().toString()).append(ExcelTypeEnum.XLSX.getValue());
    File tmpFile = new File(fileName.toString());
    writeExcelFile(tmpFile, tables, request);
    // Upload to HDFS
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(request.getClusterName());
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("Cluster info " + "[" + request.getClusterName() + "]" + "{&DOES_NOT_EXIST}");
    }
    // send request to get dbs
    String url = UriBuilder.fromUri(clusterInfo.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getUpload()).toString();
    CloseableHttpClient httpclient = HttpClients.createDefault();
    HttpPost httppost = new HttpPost(url);
    MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
    multipartEntityBuilder.setContentType(ContentType.MULTIPART_FORM_DATA);
    multipartEntityBuilder.setCharset(Charset.forName("UTF-8"));
    if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
        multipartEntityBuilder.addTextBody("path", linkisConfig.getUploadPrefix() + LINKIS_ONE_UPLOAD_PREFIX + request.getHdfsPath());
    } else {
        multipartEntityBuilder.addTextBody("path", linkisConfig.getUploadPrefix() + request.getHdfsPath());
    }
    multipartEntityBuilder.addBinaryBody("file", tmpFile);
    httppost.addHeader("Token-User", user.getUserName());
    httppost.addHeader("Token-Code", clusterInfo.getLinkisToken());
    httppost.setEntity(multipartEntityBuilder.build());
    CloseableHttpResponse response = null;
    try {
        response = httpclient.execute(httppost);
    } catch (IOException e) {
        throw new UnExpectedRequestException("{&FAILED_TO_CALL_UPLOAD_API}");
    }
    int code = response.getStatusLine().getStatusCode();
    response.close();
    if (code != HttpStatus.SC_OK) {
        throw new UnExpectedRequestException("{&FAILED_TO_CALL_UPLOAD_API}");
    }
    boolean tmpDeleteFlag = tmpFile.delete();
    LOGGER.info("Delete tmp excel: " + tmpDeleteFlag);
    return new GeneralResponse<>(code + "", "{&SUCCESS_TO_UPLOAD_ANALYSIS_EXCEL}", code);
}

Example 73 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterDataSourceApplication.

@Override
public GeneralResponse<?> filterDataSourceApplication(FilterDataSourceRequest request) throws UnExpectedRequestException {
    // Check arguments
    FilterDataSourceRequest.checkRequest(request);
    Integer page = request.getPage();
    Integer size = request.getSize();
    String clusterName = request.getClusterName();
    String databaseName = StringUtils.isEmpty(request.getDatabaseName()) ? "" : request.getDatabaseName();
    String tableName = StringUtils.isEmpty(request.getTableName()) ? "" : request.getTableName();
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    List<TaskDataSource> taskDataSources;
    long total;
    // Find datasource by user
    taskDataSources = taskDataSourceDao.findByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName, page, size);
    total = taskDataSourceDao.countByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName);
    List<Application> applicationList = taskDataSources.stream().map(jobDataSource -> jobDataSource.getTask().getApplication()).collect(Collectors.toList());
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}

Example 74 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class JobServiceImpl method getTaskLog.

@Override
public GeneralResponse<?> getTaskLog(Long taskId, String clusterName) throws UnExpectedRequestException {
    Task task = taskDao.findById(taskId);
    if (task == null) {
        throw new UnExpectedRequestException("{&JOB_ID_DOES_NOT_EXIST}");
    }
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("Cluster info {&DOES_NOT_EXIST}");
    }
    LogResult logResult;
    String proxyUser = task.getTaskProxyUser();
    try {
        logResult = monitorManager.getTaskPartialLog(task.getTaskRemoteId(), 0, StringUtils.isNotBlank(proxyUser) ? proxyUser : task.getApplication().getExecuteUser(), clusterInfo.getLinkisAddress(), clusterName);
    } catch (LogPartialException | ClusterInfoNotConfigException e) {
        throw new UnExpectedRequestException(e.getMessage());
    }
    LOGGER.info("Succeed to get task log, task_id: {}, cluster_id: {}", taskId, clusterName);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_LOG}", logResult.getLog());
}

Example 75 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class TaskServiceImpl method getTaskDetail.

@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
    Task taskInDb = taskDao.findById(taskId);
    if (taskInDb == null) {
        throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
    }
    // FIXME:
    List<TaskRuleAlarmConfig> distinct;
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
        taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
        taskRuleSimples.add(taskRuleSimple);
    }
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Find single table verification rules
    List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of single table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
        singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find custom table verification rules
    List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of custom table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : customRuleIds) {
        customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find multi-table verification rules
    List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of multi-table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
        multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find file table verification rules
    List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of file table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
        fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    List<Long> allRuleIds = new ArrayList<>();
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        allRuleIds.add(taskRuleSimple.getRuleId());
        if (taskRuleSimple.getChildRuleSimple() != null) {
            allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
        }
    }
    List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
    Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
    for (TaskResult taskResult : allTaskResult) {
        if (allResultMap.get(taskResult.getRuleId()) != null) {
            allResultMap.get(taskResult.getRuleId()).add(taskResult);
        } else {
            List<TaskResult> taskResults = new ArrayList<>(1);
            taskResults.add(taskResult);
            allResultMap.put(taskResult.getRuleId(), taskResults);
        }
    }
    TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}