Search in sources :

Example 71 with GeneralResponse

use of com.webank.wedatasphere.qualitis.response.GeneralResponse in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method getDataSource.

@Override
public GeneralResponse<?> getDataSource(PageRequest request) throws UnExpectedRequestException {
    // Check arguments
    PageRequest.checkRequest(request);
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    long total;
    // Find datasource by user
    List<Map<String, String>> taskDataSourceMap = taskDataSourceDao.findByUser(user.getUserName());
    total = taskDataSourceMap.size();
    List<ApplicationClusterResponse> response = new ArrayList<>();
    Map<String, ApplicationClusterResponse> map = new HashMap<>(2);
    for (Map<String, String> currentMap : taskDataSourceMap) {
        putIntoCluster(response, currentMap, map);
    }
    LOGGER.info("Succeed to find dataSources. size: {}, id of dataSources: {}", total, response);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_DATASOURCE}", response);
}
Also used : User(com.webank.wedatasphere.qualitis.entity.User) ApplicationClusterResponse(com.webank.wedatasphere.qualitis.response.ApplicationClusterResponse) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) Map(java.util.Map) HashMap(java.util.HashMap)

Example 72 with GeneralResponse

use of com.webank.wedatasphere.qualitis.response.GeneralResponse in project Qualitis by WeBankFinTech.

the class Filter1AuthorizationFilter method writeRedirectHome.

private void writeRedirectHome(ServletResponse response) throws IOException {
    ((HttpServletResponse) response).setStatus(Response.Status.UNAUTHORIZED.getStatusCode());
    ServletOutputStream out = response.getOutputStream();
    GeneralResponse generalResponse = new GeneralResponse<>("401", "please login", null);
    out.write(objectMapper.writeValueAsBytes(generalResponse));
    out.flush();
}
Also used : GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) HttpServletResponse(javax.servlet.http.HttpServletResponse)

Example 73 with GeneralResponse

use of com.webank.wedatasphere.qualitis.response.GeneralResponse in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method uploadDataSourceAnalysisResult.

@Override
public GeneralResponse<?> uploadDataSourceAnalysisResult(UploadResultRequest request) throws UnExpectedRequestException, IOException {
    // Login user permission.
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    if (user == null) {
        throw new UnExpectedRequestException("User {&DOES_NOT_EXIST}");
    }
    UploadResultRequest.checkRequest(request);
    List<ApplicationClusterResponse> responses = (List<ApplicationClusterResponse>) getDataSource(new PageRequest(0, Integer.MAX_VALUE)).getData();
    List<String> tables = responses.stream().filter(cluster -> cluster.getClusterName().equals(request.getClusterName())).map(ApplicationClusterResponse::getDatabase).flatMap(database -> database.stream()).filter(databaseResponse -> databaseResponse.getDatabaseName().equals(request.getDatabaseName())).map(ApplicationDatabaseResponse::getTable).flatMap(table -> table.stream()).distinct().collect(Collectors.toList());
    if (StringUtils.isNotBlank(request.getTableName())) {
        tables.clear();
        tables.add(request.getTableName());
    }
    LOGGER.info("Start to write excel");
    StringBuffer fileName = new StringBuffer();
    fileName.append(linkisConfig.getUploadTmpPath()).append(File.separator).append(user.getUserName()).append("_").append(request.getClusterName()).append("_").append(request.getDatabaseName()).append("_").append(UUID.randomUUID().toString()).append(ExcelTypeEnum.XLSX.getValue());
    File tmpFile = new File(fileName.toString());
    writeExcelFile(tmpFile, tables, request);
    // Upload to HDFS
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(request.getClusterName());
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("Cluster info " + "[" + request.getClusterName() + "]" + "{&DOES_NOT_EXIST}");
    }
    // send request to get dbs
    String url = UriBuilder.fromUri(clusterInfo.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getUpload()).toString();
    CloseableHttpClient httpclient = HttpClients.createDefault();
    HttpPost httppost = new HttpPost(url);
    MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
    multipartEntityBuilder.setContentType(ContentType.MULTIPART_FORM_DATA);
    multipartEntityBuilder.setCharset(Charset.forName("UTF-8"));
    if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
        multipartEntityBuilder.addTextBody("path", linkisConfig.getUploadPrefix() + LINKIS_ONE_UPLOAD_PREFIX + request.getHdfsPath());
    } else {
        multipartEntityBuilder.addTextBody("path", linkisConfig.getUploadPrefix() + request.getHdfsPath());
    }
    multipartEntityBuilder.addBinaryBody("file", tmpFile);
    httppost.addHeader("Token-User", user.getUserName());
    httppost.addHeader("Token-Code", clusterInfo.getLinkisToken());
    httppost.setEntity(multipartEntityBuilder.build());
    CloseableHttpResponse response = null;
    try {
        response = httpclient.execute(httppost);
    } catch (IOException e) {
        throw new UnExpectedRequestException("{&FAILED_TO_CALL_UPLOAD_API}");
    }
    int code = response.getStatusLine().getStatusCode();
    response.close();
    if (code != HttpStatus.SC_OK) {
        throw new UnExpectedRequestException("{&FAILED_TO_CALL_UPLOAD_API}");
    }
    boolean tmpDeleteFlag = tmpFile.delete();
    LOGGER.info("Delete tmp excel: " + tmpDeleteFlag);
    return new GeneralResponse<>(code + "", "{&SUCCESS_TO_UPLOAD_ANALYSIS_EXCEL}", code);
}
Also used : HttpPost(org.apache.http.client.methods.HttpPost) AlarmConfigStatusEnum(com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum) StringUtils(org.apache.commons.lang.StringUtils) LinkisConfig(com.webank.wedatasphere.qualitis.config.LinkisConfig) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) Autowired(org.springframework.beans.factory.annotation.Autowired) Application(com.webank.wedatasphere.qualitis.entity.Application) ProjectUserDao(com.webank.wedatasphere.qualitis.project.dao.ProjectUserDao) ApplicationDatabaseResponse(com.webank.wedatasphere.qualitis.response.ApplicationDatabaseResponse) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) FilterDataSourceRequest(com.webank.wedatasphere.qualitis.request.FilterDataSourceRequest) HttpUtils(com.webank.wedatasphere.qualitis.util.HttpUtils) Task(com.webank.wedatasphere.qualitis.entity.Task) TaskRuleAlarmConfig(com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig) Map(java.util.Map) ApplicationClusterResponse(com.webank.wedatasphere.qualitis.response.ApplicationClusterResponse) UriBuilder(javax.ws.rs.core.UriBuilder) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Path(java.nio.file.Path) RestTemplate(org.springframework.web.client.RestTemplate) Context(javax.ws.rs.core.Context) MultipartEntityBuilder(org.apache.http.entity.mime.MultipartEntityBuilder) PageRequest(com.webank.wedatasphere.qualitis.request.PageRequest) UploadResultRequest(com.webank.wedatasphere.qualitis.request.UploadResultRequest) FilterAdvanceRequest(com.webank.wedatasphere.qualitis.request.FilterAdvanceRequest) ContentType(org.apache.http.entity.ContentType) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) TaskRuleSimpleDao(com.webank.wedatasphere.qualitis.dao.TaskRuleSimpleDao) ExcelTypeEnum(com.alibaba.excel.support.ExcelTypeEnum) List(java.util.List) Sheet(com.alibaba.excel.metadata.Sheet) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) HttpClients(org.apache.http.impl.client.HttpClients) BeanUtils(org.springframework.beans.BeanUtils) TaskStatusEnum(com.webank.wedatasphere.qualitis.constant.TaskStatusEnum) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) HashMap(java.util.HashMap) FilterProjectRequest(com.webank.wedatasphere.qualitis.request.FilterProjectRequest) FilterStatusRequest(com.webank.wedatasphere.qualitis.request.FilterStatusRequest) ProjectDao(com.webank.wedatasphere.qualitis.project.dao.ProjectDao) ArrayList(java.util.ArrayList) ExcelWriter(com.alibaba.excel.ExcelWriter) HttpServletRequest(javax.servlet.http.HttpServletRequest) User(com.webank.wedatasphere.qualitis.entity.User) CheckTemplateEnum(com.webank.wedatasphere.qualitis.rule.constant.CheckTemplateEnum) Charset(java.nio.charset.Charset) TaskDao(com.webank.wedatasphere.qualitis.dao.TaskDao) CloseableHttpResponse(org.apache.http.client.methods.CloseableHttpResponse) Service(org.springframework.stereotype.Service) ApplicationStatusEnum(com.webank.wedatasphere.qualitis.constant.ApplicationStatusEnum) CompareTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.CompareTypeEnum) ApplicationService(com.webank.wedatasphere.qualitis.service.ApplicationService) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) OutputStream(java.io.OutputStream) CloseableHttpClient(org.apache.http.impl.client.CloseableHttpClient) Logger(org.slf4j.Logger) Files(java.nio.file.Files) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) TaskDataSourceDao(com.webank.wedatasphere.qualitis.dao.TaskDataSourceDao) ExcelResult(com.webank.wedatasphere.qualitis.excel.ExcelResult) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) File(java.io.File) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) ExcelSheetName(com.webank.wedatasphere.qualitis.project.constant.ExcelSheetName) Paths(java.nio.file.Paths) ApplicationDao(com.webank.wedatasphere.qualitis.dao.ApplicationDao) UserDao(com.webank.wedatasphere.qualitis.dao.UserDao) TaskResultDao(com.webank.wedatasphere.qualitis.dao.TaskResultDao) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) CloseableHttpClient(org.apache.http.impl.client.CloseableHttpClient) HttpPost(org.apache.http.client.methods.HttpPost) User(com.webank.wedatasphere.qualitis.entity.User) MultipartEntityBuilder(org.apache.http.entity.mime.MultipartEntityBuilder) ApplicationClusterResponse(com.webank.wedatasphere.qualitis.response.ApplicationClusterResponse) IOException(java.io.IOException) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) PageRequest(com.webank.wedatasphere.qualitis.request.PageRequest) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) CloseableHttpResponse(org.apache.http.client.methods.CloseableHttpResponse) List(java.util.List) ArrayList(java.util.ArrayList) File(java.io.File)

Example 74 with GeneralResponse

use of com.webank.wedatasphere.qualitis.response.GeneralResponse in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterApplicationId.

/**
 * Find application by applicationId
 * @param applicationId
 * @return
 */
@Override
public GeneralResponse<?> filterApplicationId(String applicationId) {
    Long userId = HttpUtils.getUserId(httpServletRequest);
    // Find applications by user
    User user = userDao.findById(userId);
    List<Application> applicationList;
    applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), applicationId);
    if (applicationList == null) {
        LOGGER.info("User: {} , Not find applications with applicationId: {}", user.getUserName(), applicationId);
        return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS_BUT_FIND_NO_RESULTS}", null);
    }
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(applicationList.size());
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("User: {}, find {} applications with like applicationId : {},Id of applications: {}", user.getUserName(), applicationList.size(), applicationId, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) User(com.webank.wedatasphere.qualitis.entity.User) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Application(com.webank.wedatasphere.qualitis.entity.Application)

Example 75 with GeneralResponse

use of com.webank.wedatasphere.qualitis.response.GeneralResponse in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterAdvanceApplication.

@Override
public GeneralResponse<?> filterAdvanceApplication(FilterAdvanceRequest request) {
    Long userId = HttpUtils.getUserId(httpServletRequest);
    // Find applications by user
    User user = userDao.findById(userId);
    List<Application> applicationList;
    long total = 0;
    // If application ID is not empty, just return application which ID like the input string.
    if (StringUtils.isNotBlank(request.getApplicationId())) {
        applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), request.getApplicationId());
        total = applicationList.size();
    } else if (StringUtils.isNotBlank(request.getClusterName())) {
        if (request.getStatus() != null) {
            if (request.getStatus().equals(ApplicationStatusEnum.FINISHED.getCode())) {
                request.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.NOT_PASS.getCode())) {
                request.setStatus(TaskStatusEnum.FAIL_CHECKOUT.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.SUCCESSFUL_CREATE_APPLICATION.getCode())) {
                request.setStatus(TaskStatusEnum.INITED.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.RUNNING.getCode())) {
                request.setStatus(TaskStatusEnum.RUNNING.getCode());
            } else if (request.getStatus() == 0) {
                request.setStatus(null);
            }
        }
        // If data source is not empty, it will be used as the basic filter.
        applicationList = applicationDao.findApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
        total = applicationDao.countApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
    } else {
        applicationList = applicationDao.findApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
        total = applicationDao.countApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
    }
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) User(com.webank.wedatasphere.qualitis.entity.User) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Application(com.webank.wedatasphere.qualitis.entity.Application)

Aggregations

GeneralResponse (com.webank.wedatasphere.qualitis.response.GeneralResponse)146 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)96 Transactional (org.springframework.transaction.annotation.Transactional)49 ArrayList (java.util.ArrayList)48 User (com.webank.wedatasphere.qualitis.entity.User)40 GetAllResponse (com.webank.wedatasphere.qualitis.response.GetAllResponse)30 Project (com.webank.wedatasphere.qualitis.project.entity.Project)28 Map (java.util.Map)27 ClusterInfo (com.webank.wedatasphere.qualitis.entity.ClusterInfo)26 MetaDataAcquireFailedException (com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException)24 List (java.util.List)22 Rule (com.webank.wedatasphere.qualitis.rule.entity.Rule)21 PermissionDeniedRequestException (com.webank.wedatasphere.qualitis.exception.PermissionDeniedRequestException)17 IOException (java.io.IOException)16 UserRole (com.webank.wedatasphere.qualitis.entity.UserRole)15 Date (java.util.Date)15 Task (com.webank.wedatasphere.qualitis.entity.Task)14 JSONObject (org.json.JSONObject)14 HttpEntity (org.springframework.http.HttpEntity)14 HttpHeaders (org.springframework.http.HttpHeaders)14