Examples with BatchCatalogue com.dtstack.taier.dao.domain.BatchCatalogue used on opensource projects

Example 1 with BatchCatalogue

use of com.dtstack.taier.dao.domain.BatchCatalogue in project Taier by DTStack.

the class BatchResourceService method addResource.

/**
 * 添加资源
 */
public CatalogueVO addResource(BatchResourceAddDTO batchResourceAddDTO) {
    Long tenantId = batchResourceAddDTO.getTenantId();
    Long userId = batchResourceAddDTO.getUserId();
    String resourceName;
    Long resourceId = null;
    BatchResource resourceDB = null;
    Integer resourceType = null;
    if (batchResourceAddDTO.getId() != null && batchResourceAddDTO.getId() != 0L) {
        resourceId = batchResourceAddDTO.getId();
        resourceDB = this.developResourceDao.getOne(resourceId);
        resourceName = resourceDB.getResourceName();
        resourceType = resourceDB.getResourceType();
    } else {
        if (StringUtils.isEmpty(batchResourceAddDTO.getResourceName())) {
            throw new RdosDefineException("需要设置参数 resourceName.", ErrorCode.INVALID_PARAMETERS);
        }
        resourceName = batchResourceAddDTO.getResourceName();
        resourceType = batchResourceAddDTO.getResourceType() == null ? ResourceType.OTHER.getType() : batchResourceAddDTO.getResourceType();
    }
    String hdfsPath = uploadHDFSFileWithResource(tenantId, resourceName, batchResourceAddDTO.getOriginalFilename(), batchResourceAddDTO.getTmpPath());
    BatchResource batchResource = null;
    // 重新上传资源
    if (Objects.nonNull(resourceId)) {
        batchResource = resourceDB;
        if (Deleted.DELETED.getStatus().equals(batchResource.getIsDeleted())) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_RESOURCE);
        }
        batchResource.setResourceDesc(batchResourceAddDTO.getResourceDesc());
        batchResource.setOriginFileName(batchResourceAddDTO.getOriginalFilename());
        batchResource.setUrl(hdfsPath);
    } else {
        // 判断是否已经存在相同的资源了
        batchTaskService.checkName(resourceName, CatalogueType.RESOURCE_MANAGER.name(), null, 1, tenantId);
        batchResourceAddDTO.setUrl(hdfsPath);
        batchResourceAddDTO.setCreateUserId(userId);
        batchResource = PublicUtil.objectToObject(batchResourceAddDTO, BatchResource.class);
        if (Objects.isNull(batchResource)) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_RESOURCE);
        }
        batchResource.setOriginFileName(batchResourceAddDTO.getOriginalFilename());
    }
    // resourceType 设置默认值
    resourceType = resourceType != null ? resourceType : ResourceType.OTHER.getType();
    batchResource.setResourceType(resourceType);
    batchResource.setModifyUserId(userId);
    addOrUpdate(batchResource);
    BatchCatalogue catalogue = batchCatalogueService.getOne(batchResource.getNodePid());
    CatalogueVO catalogueVO = new CatalogueVO();
    catalogueVO.setId(batchResource.getId());
    catalogueVO.setName(batchResource.getResourceName());
    catalogueVO.setType("file");
    catalogueVO.setLevel(catalogue.getLevel() + 1);
    catalogueVO.setChildren(null);
    catalogueVO.setParentId(catalogue.getId());
    catalogueVO.setResourceType(resourceType);
    String username = userService.getUserName(catalogue.getCreateUserId());
    catalogueVO.setCreateUser(username);
    return catalogueVO;
}

Example 2 with BatchCatalogue

use of com.dtstack.taier.dao.domain.BatchCatalogue in project Taier by DTStack.

the class BatchCatalogueService method initTemplateCatalogue.

/**
 * 初始化任务开发下的 模板目录 和 模板任务
 * @param oneCatalogue
 * @param tenantId
 * @param userId
 * @return
 */
private List<BatchCatalogue> initTemplateCatalogue(BatchCatalogue oneCatalogue, Long tenantId, Long userId) {
    List<BatchCatalogue> templateCatalogueList = new ArrayList<>(TemplateCatalogue.getValues().size());
    // 需要初始化的模板任务所在的目录
    BatchCatalogue batchTempTaskCatalogue = new BatchCatalogue();
    batchTempTaskCatalogue.setLevel(CatalogueLevel.OTHER.getLevel());
    batchTempTaskCatalogue.setNodePid(oneCatalogue.getId());
    batchTempTaskCatalogue.setTenantId(tenantId);
    batchTempTaskCatalogue.setCreateUserId(userId);
    List<TemplateCatalogue> templateTaskCatalogues = TemplateCatalogue.getValues();
    HashMap<String, Long> idsMap = new HashMap<>();
    for (TemplateCatalogue templateCatalogue : templateTaskCatalogues) {
        // 相同目录只创建一次
        if (!idsMap.containsKey(templateCatalogue.getValue())) {
            batchTempTaskCatalogue.setNodeName(templateCatalogue.getValue());
            batchTempTaskCatalogue.setId(0L);
            batchTempTaskCatalogue.setCatalogueType(RdosBatchCatalogueTypeEnum.NORAML.getType());
            addOrUpdate(batchTempTaskCatalogue);
            idsMap.put(templateCatalogue.getValue(), batchTempTaskCatalogue.getId());
        }
        templateCatalogueList.add(batchTempTaskCatalogue);
        try {
            String content = batchTaskTemplateService.getContentByType(EScheduleJobType.SPARK_SQL.getVal(), templateCatalogue.getType());
            // 初始化任务
            TaskResourceParam batchTask = new TaskResourceParam();
            batchTask.setName(templateCatalogue.getFileName());
            batchTask.setTaskType(EScheduleJobType.SPARK_SQL.getVal());
            batchTask.setNodePid(idsMap.get(templateCatalogue.getValue()));
            batchTask.setComputeType(ComputeType.BATCH.getType());
            batchTask.setLockVersion(0);
            batchTask.setVersion(0);
            batchTask.setTenantId(tenantId);
            batchTask.setUserId(userId);
            batchTask.setCreateUserId(userId);
            batchTask.setModifyUserId(userId);
            if (StringUtils.isEmpty(content)) {
                throw new RdosDefineException(ErrorCode.TEMPLATE_TASK_CONTENT_NOT_NULL);
            }
            batchTask.setSqlText(content);
            // 添加init脚本中带有的任务参数
            List<Map> taskVariables = new ArrayList<>();
            Map<String, String> variable = new HashMap<>(3);
            variable.put("paramCommand", PARAM_COMMAND);
            variable.put("paramName", PARAM_NAME);
            variable.put("type", TYPE4SYSTEM);
            taskVariables.add(variable);
            batchTask.setTaskVariables(taskVariables);
            // SparkSQL任务支持多版本运行，添加默认的组件版本
            List<BatchTaskGetComponentVersionResultVO> hadoopVersions = batchTaskService.getComponentVersionByTaskType(tenantId, EScheduleJobType.SPARK_SQL.getVal());
            if (CollectionUtils.isNotEmpty(hadoopVersions)) {
                batchTask.setComponentVersion(hadoopVersions.get(0).getComponentVersion());
            }
            batchTaskService.addOrUpdateTask(batchTask);
        } catch (Exception e) {
            LOGGER.error(ErrorCode.CATALOGUE_INIT_FAILED.getDescription(), e);
            throw new RdosDefineException(ErrorCode.CATALOGUE_INIT_FAILED);
        }
    }
    return templateCatalogueList;
}

Example 3 with BatchCatalogue

use of com.dtstack.taier.dao.domain.BatchCatalogue in project Taier by DTStack.

the class BatchTaskService method getTaskById.

/**
 * 数据开发-根据任务id，查询详情
 *
 * @return
 * @author toutian
 */
public BatchTaskBatchVO getTaskById(final ScheduleTaskVO scheduleTaskVO) {
    final BatchTask task = this.developTaskDao.getOne(scheduleTaskVO.getId());
    if (task == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    final List<BatchResource> resources = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.MAIN_RES.getType());
    final List<BatchResource> refResourceIdList = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.DEPENDENCY_RES.getType());
    final BatchTaskBatchVO taskVO = new BatchTaskBatchVO(this.batchTaskTaskService.getForefathers(task));
    taskVO.setVersion(task.getVersion());
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        // 同步任务类型
        final String taskJson = Base64Util.baseDecode(task.getSqlText());
        if (StringUtils.isBlank(taskJson)) {
            // 向导模式存在为空的情况
            taskVO.setCreateModel(Constant.CREATE_MODEL_GUIDE);
            taskVO.setSqlText("");
        } else {
            final JSONObject obj = JSON.parseObject(taskJson);
            taskVO.setCreateModel(obj.get("createModel") == null ? Constant.CREATE_MODEL_GUIDE : Integer.parseInt(String.valueOf(obj.get("createModel"))));
            formatSqlText(taskVO, obj);
        }
    }
    this.setTaskOperatorModelAndOptions(taskVO, task);
    if (task.getFlowId() != null && task.getFlowId() > 0) {
        taskVO.setFlowId(task.getFlowId());
        final BatchTask flow = this.developTaskDao.getOne(task.getFlowId());
        if (flow != null) {
            taskVO.setFlowName(flow.getName());
        }
    }
    final BatchCatalogue catalogue = batchCatalogueService.getOne(task.getNodePid());
    if (catalogue != null) {
        taskVO.setNodePName(catalogue.getNodeName());
    }
    taskVO.setResourceList(resources);
    taskVO.setRefResourceList(refResourceIdList);
    PageQuery pageQuery = new PageQuery(1, 5, "gmt_create", Sort.DESC.name());
    List<BatchTaskVersionDetailDTO> taskVersions = batchTaskVersionService.listByTaskId(scheduleTaskVO.getId(), pageQuery).stream().map(ver -> {
        if (StringUtils.isNotBlank(ver.getOriginSql())) {
            if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
                ver.setSqlText(ver.getSqlText());
            } else {
                ver.setSqlText(ver.getOriginSql());
            }
        }
        // 填充用户名称
        ver.setUserName(userService.getUserName(ver.getCreateUserId()));
        return ver;
    }).collect(Collectors.toList());
    taskVO.setTaskVersions(taskVersions);
    // 密码脱敏 --2019/10/25 茂茂-- 同步任务 密码脱敏 仅 向导模式 修改成 全部模式
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        try {
            taskVO.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVO.getSqlText())));
        } catch (final Exception e) {
            LOGGER.error("同步任务json解析失败 taskId = {}", task.getId(), e);
            taskVO.setSqlText(DataFilter.passwordFilter(taskVO.getSqlText()));
        }
        for (BatchTaskVersionDetailDTO taskVersion : taskVO.getTaskVersions()) {
            try {
                taskVersion.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVersion.getSqlText())));
            } catch (final Exception e) {
                LOGGER.error("同步任务json解析失败 taskVersionId = {}", taskVersion.getId(), e);
                taskVersion.setSqlText(DataFilter.passwordFilter(taskVersion.getSqlText()));
            }
        }
    }
    final ReadWriteLockVO readWriteLockVO = this.readWriteLockService.getDetail(scheduleTaskVO.getTenantId(), task.getId(), ReadWriteLockType.BATCH_TASK, scheduleTaskVO.getUserId(), task.getModifyUserId(), task.getGmtModified());
    taskVO.setReadWriteLockVO(readWriteLockVO);
    taskVO.setUserId(scheduleTaskVO.getUserId());
    setTaskVariables(taskVO, scheduleTaskVO.getId());
    final List<Long> userIds = new ArrayList<>();
    userIds.add(task.getCreateUserId());
    final Map<Long, User> userMap = userService.getUserMap(userIds);
    buildUserDTOInfo(userMap, taskVO);
    return taskVO;
}

Example 4 with BatchCatalogue

use of com.dtstack.taier.dao.domain.BatchCatalogue in project Taier by DTStack.

the class BatchCatalogueService method getChildNode.

/**
 * 获得当前节点的子节点信息，包括子孙文件夹和子孙文件
 *
 * @param tenantId   租户id
 * @param isGetFile
 * @param userId
 * @return
 * @author jiangbo、toutian
 */
private CatalogueVO getChildNode(CatalogueVO currentCatalogueVO, Boolean isGetFile, Long userId, Long tenantId) {
    BatchCatalogue currentCatalogue = developCatalogueDao.getOne(currentCatalogueVO.getId());
    if (currentCatalogue == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_CATALOGUE);
    }
    currentCatalogueVO.setTenantId(currentCatalogue.getTenantId());
    currentCatalogueVO.setName(currentCatalogue.getNodeName());
    currentCatalogueVO.setLevel(currentCatalogue.getLevel());
    currentCatalogueVO.setParentId(currentCatalogue.getNodePid());
    currentCatalogueVO.setType(FILE_TYPE_FOLDER);
    // 获取目录下的资源或任务列表
    if (isGetFile) {
        // 目录下的文件信息
        List<CatalogueVO> catalogueChildFileList = Lists.newArrayList();
        // 用户id 和 名称映射
        Map<Long, String> userIdAndNameMap = Maps.newHashMap();
        // 任务目录
        if (CatalogueType.TASK_DEVELOP.getType().equals(currentCatalogueVO.getCatalogueType())) {
            List<BatchTask> taskList = batchTaskService.catalogueListBatchTaskByNodePid(tenantId, currentCatalogueVO.getId());
            taskList.sort(Comparator.comparing(BatchTask::getName));
            if (CollectionUtils.isNotEmpty(taskList)) {
                List<Long> taskIds = taskList.stream().map(BatchTask::getId).collect(Collectors.toList());
                Map<Long, ReadWriteLockVO> readWriteLockIdAndVOMap = getReadWriteLockVOMap(tenantId, taskIds, userId, userIdAndNameMap);
                // 遍历目录下的所有任务
                for (BatchTask task : taskList) {
                    CatalogueVO childCatalogueTask = new CatalogueVO();
                    BeanUtils.copyProperties(task, childCatalogueTask);
                    childCatalogueTask.setType("file");
                    childCatalogueTask.setLevel(currentCatalogueVO.getLevel() + 1);
                    childCatalogueTask.setParentId(currentCatalogueVO.getId());
                    childCatalogueTask.setCreateUser(getUserNameInMemory(userIdAndNameMap, task.getCreateUserId()));
                    // 设置任务的读写锁信息
                    ReadWriteLockVO readWriteLockVO = readWriteLockIdAndVOMap.get(task.getId());
                    if (readWriteLockVO.getLastKeepLockUserName() == null) {
                        readWriteLockVO.setLastKeepLockUserName(getUserNameInMemory(userIdAndNameMap, task.getModifyUserId()));
                        readWriteLockVO.setGmtModified(task.getGmtModified());
                    }
                    childCatalogueTask.setReadWriteLockVO(readWriteLockVO);
                    catalogueChildFileList.add(childCatalogueTask);
                }
            }
        } else if (FUNCTION_CATALOGUE_TYPE.contains(currentCatalogueVO.getCatalogueType())) {
            // 处理函数目录
            List<BatchFunction> functionList = batchFunctionService.listByNodePidAndTenantId(currentCatalogueVO.getTenantId(), currentCatalogueVO.getId());
            if (CollectionUtils.isNotEmpty(functionList)) {
                functionList.sort(Comparator.comparing(BatchFunction::getName));
                for (BatchFunction function : functionList) {
                    CatalogueVO child = new CatalogueVO();
                    BeanUtils.copyProperties(function, child);
                    child.setLevel(currentCatalogueVO.getLevel() + 1);
                    child.setType("file");
                    child.setCreateUser(getUserNameInMemory(userIdAndNameMap, function.getCreateUserId()));
                    child.setParentId(function.getNodePid());
                    catalogueChildFileList.add(child);
                }
            }
        } else if (CatalogueType.RESOURCE_MANAGER.getType().equals(currentCatalogueVO.getCatalogueType())) {
            // 处理资源目录
            List<BatchResource> resourceList = batchResourceService.listByPidAndTenantId(tenantId, currentCatalogueVO.getId());
            resourceList.sort(Comparator.comparing(BatchResource::getResourceName));
            if (CollectionUtils.isNotEmpty(resourceList)) {
                for (BatchResource resource : resourceList) {
                    CatalogueVO childResource = new CatalogueVO();
                    BeanUtils.copyProperties(resource, childResource);
                    childResource.setName(resource.getResourceName());
                    childResource.setType("file");
                    childResource.setLevel(currentCatalogueVO.getLevel() + 1);
                    childResource.setParentId(currentCatalogueVO.getId());
                    childResource.setCreateUser(getUserNameInMemory(userIdAndNameMap, resource.getCreateUserId()));
                    catalogueChildFileList.add(childResource);
                }
            }
        }
        currentCatalogueVO.setChildren(catalogueChildFileList);
    }
    // 获取目录下的子目录
    List<BatchCatalogue> childCatalogues = this.getChildCataloguesByType(currentCatalogueVO.getId(), currentCatalogueVO.getCatalogueType(), currentCatalogue.getTenantId());
    childCatalogues = keepInitCatalogueBeTop(childCatalogues, currentCatalogue);
    List<CatalogueVO> children = new ArrayList<>();
    for (BatchCatalogue catalogue : childCatalogues) {
        CatalogueVO cv = CatalogueVO.toVO(catalogue);
        cv.setType(FILE_TYPE_FOLDER);
        this.changeSQLFunctionCatalogueType(catalogue, cv, currentCatalogueVO);
        children.add(cv);
    }
    if (currentCatalogueVO.getChildren() == null) {
        currentCatalogueVO.setChildren(children);
    } else {
        currentCatalogueVO.getChildren().addAll(0, children);
    }
    return currentCatalogueVO;
}

Example 5 with BatchCatalogue

use of com.dtstack.taier.dao.domain.BatchCatalogue in project Taier by DTStack.

the class BatchCatalogueService method initEngineCatalogue.

/**
 *  初始化函数相关的二级菜单，系统函数目录 和 自定义函数目录
 * @param tenantId
 * @param userId
 * @param name
 * @param oneBatchCatalogue
 */
private void initEngineCatalogue(Long tenantId, Long userId, String name, BatchCatalogue oneBatchCatalogue) {
    // 一级菜单初始化的时候  函数管理的一级菜单为引擎 原有的一级菜单 系统函数 自定义函数 挂在引擎下 作为二级菜单
    if (isNeedFunction(name)) {
        // 离线需要初始化的函数目录
        List<Dict> batchFunctionDictList = dictService.listByDictType(DictType.DATA_DEVELOP_FUNCTION);
        if (CollectionUtils.isNotEmpty(batchFunctionDictList)) {
            for (Dict functionDict : batchFunctionDictList) {
                // 需要 系统函数、自定义函数 挂在当前目录下
                BatchCatalogue twoBatchCatalogue = new BatchCatalogue();
                twoBatchCatalogue.setNodeName(functionDict.getDictDesc());
                twoBatchCatalogue.setLevel(CatalogueLevel.SECOND.getLevel());
                twoBatchCatalogue.setNodePid(oneBatchCatalogue.getId());
                twoBatchCatalogue.setTenantId(tenantId);
                twoBatchCatalogue.setCreateUserId(userId);
                twoBatchCatalogue.setCatalogueType(RdosBatchCatalogueTypeEnum.NORAML.getType());
                addOrUpdate(twoBatchCatalogue);
            }
        }
    }
}