Examples with ComponentVO com.dtstack.taier.scheduler.vo.ComponentVO used on opensource projects

Example 1 with ComponentVO

use of com.dtstack.taier.scheduler.vo.ComponentVO in project Taier by DTStack.

the class TenantService method initDataDevelop.

@Transactional(rollbackFor = Exception.class)
public void initDataDevelop(Long clusterId, Long tenantId, Long userId, String tenantName, String tenantDesc, List<ComponentBindDBDTO> bindDBDTOList) throws Exception {
    // 初始化目录
    List<Component> components = componentService.listAllComponents(clusterId);
    List<ComponentVO> componentVOS = ComponentVO.toVOS(components);
    batchCatalogueService.initCatalogue(tenantId, userId, componentVOS);
    // 初始化数据源相关的信息
    IComponentService componentService = null;
    for (ComponentBindDBDTO componentBindDBDTO : bindDBDTOList) {
        EComponentType eComponentType = EComponentType.getByCode(componentBindDBDTO.getComponentCode());
        String componentIdentity = tenantName;
        // db相关的操作
        if (BooleanUtils.isTrue(componentBindDBDTO.getCreateFlag())) {
            componentService = multiEngineServiceFactory.getComponentService(eComponentType.getTypeCode());
            componentService.createDatabase(clusterId, eComponentType, componentIdentity, tenantDesc);
        } else {
            componentIdentity = componentBindDBDTO.getDbName();
        }
        // 初始化数据源
        datasourceService.initDefaultSource(clusterId, eComponentType, tenantId, componentIdentity, tenantDesc, userId);
        // 初始化租户引擎关系
        TenantComponent tenantEngine = new TenantComponent();
        tenantEngine.setTaskType(ComponentTypeToEScheduleJobMapping.getEScheduleTypeByComponentCode(eComponentType.getTypeCode()).getType());
        tenantEngine.setTenantId(tenantId);
        tenantEngine.setComponentIdentity(componentIdentity);
        tenantEngine.setCreateUserId(userId);
        tenantEngine.setStatus(0);
        developTenantComponentService.insert(tenantEngine);
    }
}

Example 2 with ComponentVO

use of com.dtstack.taier.scheduler.vo.ComponentVO in project Taier by DTStack.

the class UploadController method addOrUpdateComponent.

@PostMapping(value = "/component/addOrUpdateComponent")
public R<ComponentVO> addOrUpdateComponent(@RequestParam("resources1") List<MultipartFile> files1, @RequestParam("resources2") List<MultipartFile> files2, @RequestParam("clusterId") Long clusterId, @RequestParam(value = "componentConfig") String componentConfig, @RequestParam("versionName") @NotNull String versionName, @RequestParam("kerberosFileName") String kerberosFileName, @RequestParam("componentTemplate") String componentTemplate, @RequestParam("componentCode") Integer componentCode, @RequestParam("storeType") Integer storeType, @RequestParam("principals") String principals, @RequestParam("principal") String principal, @RequestParam(value = "isMetadata", defaultValue = "false") Boolean isMetadata, @RequestParam(value = "isDefault", defaultValue = "false") Boolean isDefault, @RequestParam(value = "deployType") Integer deployType) {
    List<Resource> resources = getResourcesFromFiles(files1);
    List<Resource> resourcesAdd = getResourcesFromFiles(files2);
    resources.addAll(resourcesAdd);
    return new APITemplate<ComponentVO>() {

        @Override
        protected void checkParams() throws IllegalArgumentException {
            if (null == componentCode) {
                throw new RdosDefineException("Component type cannot be empty");
            }
            if (null == clusterId) {
                throw new RdosDefineException("Cluster Id cannot be empty");
            }
            if (CollectionUtils.isNotEmpty(resources) && resources.size() >= 2 && StringUtils.isBlank(kerberosFileName)) {
                // 上传二份文件 需要kerberosFileName文件名字段
                throw new RdosDefineException("kerberosFileName不能为空");
            }
            // 校验引擎是否添加
            if (EComponentType.deployTypeComponents.contains(componentCode) && null == deployType) {
                throw new RdosDefineException(ErrorCode.EMPTY_PARAMETERS.getMsg() + ":deployType");
            }
        }

        @Override
        protected ComponentVO process() throws RdosDefineException {
            // 校验引擎是否添加
            EComponentType componentType = EComponentType.getByCode(componentCode);
            if (EComponentType.deployTypeComponents.contains(componentType) && null == deployType) {
                throw new RdosDefineException("deploy type cannot be empty");
            }
            return consoleComponentService.addOrUpdateComponent(clusterId, componentConfig, resources, versionName, kerberosFileName, componentTemplate, componentType, storeType, principals, principal, isMetadata, isDefault, deployType);
        }
    }.execute();
}

Example 3 with ComponentVO

use of com.dtstack.taier.scheduler.vo.ComponentVO in project Taier by DTStack.

the class ComponentConfigService method getComponentVoByComponent.

public List<IComponentVO> getComponentVoByComponent(List<Component> components, boolean isFilter, Long clusterId, boolean isConvertHadoopVersion, boolean multiVersion) {
    if (null == clusterId) {
        throw new RdosDefineException("集群id不能为空");
    }
    if (CollectionUtils.isEmpty(components)) {
        return new ArrayList<>(0);
    }
    // 集群所关联的组件的配置
    List<ComponentConfig> componentConfigs = componentConfigMapper.listByClusterId(clusterId, isFilter);
    if (CollectionUtils.isEmpty(componentConfigs)) {
        return new ArrayList<>(0);
    }
    // 组件按类型分组, 因为可能存在组件有多个版本, 此时需要兼容单版本和多版本格式问题
    Map<Integer, IComponentVO> componentVoMap = new HashMap<>(components.size());
    components.stream().collect(Collectors.groupingBy(Component::getComponentTypeCode, Collectors.toList())).forEach((k, v) -> componentVoMap.put(k, multiVersion ? ComponentMultiVersionVO.getInstanceWithCapacityAndType(k, v.size()) : ComponentVO.getInstance()));
    // 配置按照组件进行分组, 存在组件有多个版本
    Map<Long, List<ComponentConfig>> componentIdConfigs = componentConfigs.stream().collect(Collectors.groupingBy(ComponentConfig::getComponentId));
    List<IComponentVO> componentVoList = new ArrayList<>(components.size());
    for (Component component : components) {
        IComponentVO customComponent = componentVoMap.get(component.getComponentTypeCode());
        ComponentVO componentVO = IComponentVO.getComponentVo(customComponent, component);
        ;
        // 当前组件的配置
        List<ComponentConfig> configs = componentIdConfigs.get(component.getId());
        // hdfs yarn 才将自定义参数移除 过滤返回给前端
        boolean isHadoopControl = EComponentType.hadoopVersionComponents.contains(EComponentType.getByCode(component.getComponentTypeCode()));
        if (isHadoopControl) {
            // 配置按照编辑类型进行分组
            Map<String, List<ComponentConfig>> configTypeMapping = configs.stream().collect(Collectors.groupingBy(ComponentConfig::getType));
            // hdfs yarn 4.1 template只有自定义参数
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configTypeMapping.get(EFrontType.CUSTOM_CONTROL.name()))));
            // hdfs yarn 4.1 config为xml配置参数
            componentVO.setComponentConfig(JSONObject.toJSONString(ComponentConfigUtils.convertComponentConfigToMap(configTypeMapping.get(EFrontType.XML.name()))));
        } else {
            Map<String, Object> configToMap = ComponentConfigUtils.convertComponentConfigToMap(configs);
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configs)));
            componentVO.setComponentConfig(JSONObject.toJSONString(configToMap));
            componentVO.setDeployType(component.getDeployType());
        }
        if (isConvertHadoopVersion && isHadoopControl) {
            // 设置hadoopVersion 的key 如cdh 5.1.x
            ComponentConfig componentConfig = componentConfigMapper.listByKey(component.getId(), ConfigConstant.HADOOP_VERSION);
            if (null != componentConfig) {
                componentVO.setVersionValue(componentConfig.getValue());
            } else if (StringUtils.isNotBlank(component.getVersionValue())) {
                // 兼容老数据
                String dependName = "hadoop3".equalsIgnoreCase(component.getVersionValue()) || component.getVersionValue().startsWith("3") ? "Hadoop3" : "Hadoop2";
                List<Dict> hadoopVersion = dictMapper.getByDependName(DictType.HADOOP_VERSION.type, dependName);
                if (!CollectionUtils.isEmpty(hadoopVersion)) {
                    componentVO.setVersionValue(hadoopVersion.get(0).getDictName());
                }
            }
        }
        // 多版本才需要调用
        if (customComponent.multiVersion()) {
            customComponent.addComponent(componentVO);
        }
    }
    componentVoList.addAll(componentVoMap.values());
    return componentVoList;
}

Example 4 with ComponentVO

use of com.dtstack.taier.scheduler.vo.ComponentVO in project Taier by DTStack.

the class ConsoleComponentService method addOrUpdateComponent.

@Transactional(rollbackFor = Exception.class)
public ComponentVO addOrUpdateComponent(Long clusterId, String componentConfig, List<Resource> resources, String versionName, String kerberosFileName, String componentTemplate, EComponentType componentType, Integer storeType, String principals, String principal, boolean isMetadata, Boolean isDefault, Integer deployType) {
    if (StringUtils.isBlank(componentConfig)) {
        componentConfig = new JSONObject().toJSONString();
    }
    EComponentType storeComponent = null == storeType ? null : EComponentType.getByCode(storeType);
    PartCluster partCluster = clusterFactory.newImmediatelyLoadCluster(clusterId);
    Part part = partCluster.create(componentType, versionName, storeComponent, deployType);
    String versionValue = part.getVersionValue();
    String pluginName = part.getPluginName();
    Component componentDTO = new Component();
    componentDTO.setComponentTypeCode(componentType.getTypeCode());
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    String clusterName = cluster.getClusterName();
    Component addComponent = new Component();
    BeanUtils.copyProperties(componentDTO, addComponent);
    // 判断是否是更新组件, 需要校验组件版本
    com.dtstack.taier.dao.domain.Component dbComponent = componentMapper.getByClusterIdAndComponentType(clusterId, componentType.getTypeCode(), ComponentVersionUtil.isMultiVersionComponent(componentType.getTypeCode()) ? versionValue : null, deployType);
    String dbHadoopVersion = "";
    boolean isUpdate = false;
    boolean isOpenKerberos = isOpenKerberos(kerberosFileName, dbComponent);
    if (null != dbComponent) {
        // 更新
        dbHadoopVersion = dbComponent.getVersionValue();
        addComponent = dbComponent;
        isUpdate = true;
    }
    EComponentType storesComponent = this.checkStoresComponent(clusterId, storeType);
    addComponent.setStoreType(storesComponent.getTypeCode());
    addComponent.setVersionValue(versionValue);
    addComponent.setComponentName(componentType.getName());
    addComponent.setComponentTypeCode(componentType.getTypeCode());
    addComponent.setDeployType(deployType);
    if (EComponentType.HDFS == componentType) {
        // hdfs的组件和yarn组件的版本保持强一致
        com.dtstack.taier.dao.domain.Component yarnComponent = componentMapper.getByClusterIdAndComponentType(clusterId, EComponentType.YARN.getTypeCode(), null, null);
        if (null != yarnComponent) {
            versionName = yarnComponent.getVersionName();
        }
    }
    addComponent.setVersionName(versionName);
    if (StringUtils.isNotBlank(kerberosFileName)) {
        addComponent.setKerberosFileName(kerberosFileName);
    }
    changeDefault(BooleanUtils.isTrue(isDefault), clusterId, componentType, addComponent);
    // md5zip 会作为 config 表的一个属性
    String md5Key = updateResource(clusterId, componentConfig, resources, kerberosFileName, componentType.getTypeCode(), principals, principal, addComponent, dbComponent);
    addComponent.setClusterId(clusterId);
    if (isUpdate) {
        componentMapper.updateById(addComponent);
        refreshVersion(componentType, clusterId, addComponent, dbHadoopVersion);
        clusterMapper.updateGmtModified(clusterId);
    } else {
        componentMapper.insert(addComponent);
    }
    changeMetadata(componentType.getTypeCode(), isMetadata, clusterId, addComponent.getIsMetadata());
    List<ClientTemplate> clientTemplates = this.buildTemplate(componentType, componentConfig, isOpenKerberos, md5Key, componentTemplate, pluginName);
    componentConfigService.addOrUpdateComponentConfig(clientTemplates, addComponent.getId(), addComponent.getClusterId(), componentType.getTypeCode());
    // 此时不需要查询默认版本
    List<IComponentVO> componentVos = componentConfigService.getComponentVoByComponent(Lists.newArrayList(addComponent), true, clusterId, true, false);
    this.updateCache();
    if (CollectionUtils.isNotEmpty(componentVos)) {
        ComponentVO componentVO = (ComponentVO) componentVos.get(0);
        componentVO.setClusterName(clusterName);
        componentVO.setPrincipal(principal);
        componentVO.setPrincipals(principals);
        componentVO.setDeployType(deployType);
        componentVO.setIsMetadata(BooleanUtils.toInteger(isMetadata));
        return componentVO;
    }
    return null;
}

Example 5 with ComponentVO

use of com.dtstack.taier.scheduler.vo.ComponentVO in project Taier by DTStack.

the class ConsoleClusterService method convertComponentToScheduling.

private List<SchedulingVo> convertComponentToScheduling(Table<Integer, String, KerberosConfig> kerberosTable, Map<EComponentScheduleType, List<IComponentVO>> scheduleType) {
    List<SchedulingVo> schedulingVos = new ArrayList<>();
    // 为空也返回
    for (EComponentScheduleType value : EComponentScheduleType.values()) {
        SchedulingVo schedulingVo = new SchedulingVo();
        schedulingVo.setSchedulingCode(value.getType());
        schedulingVo.setSchedulingName(value.getName());
        List<IComponentVO> componentVoList = scheduleType.getOrDefault(value, Collections.emptyList());
        if (Objects.nonNull(kerberosTable) && !kerberosTable.isEmpty() && CollectionUtils.isNotEmpty(componentVoList)) {
            componentVoList.forEach(component -> {
                // 组件每个版本设置k8s参数
                for (ComponentVO componentVO : component.loadComponents()) {
                    KerberosConfig kerberosConfig;
                    EComponentType type = EComponentType.getByCode(componentVO.getComponentTypeCode());
                    if (type == EComponentType.YARN || type == EComponentType.SPARK_THRIFT || type == EComponentType.HIVE_SERVER) {
                        kerberosConfig = kerberosTable.get(type.getTypeCode(), StringUtils.EMPTY);
                    } else {
                        kerberosConfig = kerberosTable.get(componentVO.getComponentTypeCode(), StringUtils.isBlank(componentVO.getVersionValue()) ? StringUtils.EMPTY : componentVO.getVersionValue());
                    }
                    if (Objects.nonNull(kerberosConfig)) {
                        componentVO.setPrincipal(kerberosConfig.getPrincipal());
                        componentVO.setPrincipals(kerberosConfig.getPrincipals());
                        componentVO.setMergeKrb5Content(kerberosConfig.getMergeKrbContent());
                    }
                }
            });
        }
        schedulingVo.setComponents(componentVoList);
        schedulingVos.add(schedulingVo);
    }
    return schedulingVos;
}