Examples with IComponentVO com.dtstack.taier.scheduler.vo.IComponentVO used on opensource projects

Example 1 with IComponentVO

use of com.dtstack.taier.scheduler.vo.IComponentVO in project Taier by DTStack.

the class ConsoleClusterService method getConsoleClusterInfo.

/**
 * 获取集群信息详情 需要根据组件分组
 *
 * @param clusterId
 * @return
 */
public ClusterVO getConsoleClusterInfo(Long clusterId) {
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        return new ClusterVO();
    }
    ClusterVO clusterVO = ClusterVO.toVO(cluster);
    // 查询默认版本或者多个版本
    List<com.dtstack.taier.dao.domain.Component> components = componentMapper.listByClusterId(clusterId, null, false);
    List<IComponentVO> componentConfigs = componentConfigService.getComponentVoByComponent(components, true, clusterId, true, true);
    Table<Integer, String, KerberosConfig> kerberosTable = null;
    // kerberos的配置
    kerberosTable = HashBasedTable.create();
    for (KerberosConfig kerberosConfig : consoleKerberosMapper.getByClusters(clusterId)) {
        kerberosTable.put(kerberosConfig.getComponentType(), StringUtils.isBlank(kerberosConfig.getComponentVersion()) ? StringUtils.EMPTY : kerberosConfig.getComponentVersion(), kerberosConfig);
    }
    Map<EComponentScheduleType, List<IComponentVO>> scheduleType = new HashMap<>(4);
    // 组件根据用途分组(计算,资源)
    if (CollectionUtils.isNotEmpty(componentConfigs)) {
        scheduleType = componentConfigs.stream().collect(Collectors.groupingBy(c -> EComponentType.getScheduleTypeByComponent(c.getComponentTypeCode())));
    }
    List<SchedulingVo> schedulingVos = convertComponentToScheduling(kerberosTable, scheduleType);
    clusterVO.setScheduling(schedulingVos);
    clusterVO.setCanModifyMetadata(checkMetadata(clusterId, components));
    return clusterVO;
}

Example 2 with IComponentVO

use of com.dtstack.taier.scheduler.vo.IComponentVO in project Taier by DTStack.

the class ComponentConfigService method getComponentVoByComponent.

public List<IComponentVO> getComponentVoByComponent(List<Component> components, boolean isFilter, Long clusterId, boolean isConvertHadoopVersion, boolean multiVersion) {
    if (null == clusterId) {
        throw new RdosDefineException("集群id不能为空");
    }
    if (CollectionUtils.isEmpty(components)) {
        return new ArrayList<>(0);
    }
    // 集群所关联的组件的配置
    List<ComponentConfig> componentConfigs = componentConfigMapper.listByClusterId(clusterId, isFilter);
    if (CollectionUtils.isEmpty(componentConfigs)) {
        return new ArrayList<>(0);
    }
    // 组件按类型分组, 因为可能存在组件有多个版本, 此时需要兼容单版本和多版本格式问题
    Map<Integer, IComponentVO> componentVoMap = new HashMap<>(components.size());
    components.stream().collect(Collectors.groupingBy(Component::getComponentTypeCode, Collectors.toList())).forEach((k, v) -> componentVoMap.put(k, multiVersion ? ComponentMultiVersionVO.getInstanceWithCapacityAndType(k, v.size()) : ComponentVO.getInstance()));
    // 配置按照组件进行分组, 存在组件有多个版本
    Map<Long, List<ComponentConfig>> componentIdConfigs = componentConfigs.stream().collect(Collectors.groupingBy(ComponentConfig::getComponentId));
    List<IComponentVO> componentVoList = new ArrayList<>(components.size());
    for (Component component : components) {
        IComponentVO customComponent = componentVoMap.get(component.getComponentTypeCode());
        ComponentVO componentVO = IComponentVO.getComponentVo(customComponent, component);
        ;
        // 当前组件的配置
        List<ComponentConfig> configs = componentIdConfigs.get(component.getId());
        // hdfs yarn 才将自定义参数移除 过滤返回给前端
        boolean isHadoopControl = EComponentType.hadoopVersionComponents.contains(EComponentType.getByCode(component.getComponentTypeCode()));
        if (isHadoopControl) {
            // 配置按照编辑类型进行分组
            Map<String, List<ComponentConfig>> configTypeMapping = configs.stream().collect(Collectors.groupingBy(ComponentConfig::getType));
            // hdfs yarn 4.1 template只有自定义参数
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configTypeMapping.get(EFrontType.CUSTOM_CONTROL.name()))));
            // hdfs yarn 4.1 config为xml配置参数
            componentVO.setComponentConfig(JSONObject.toJSONString(ComponentConfigUtils.convertComponentConfigToMap(configTypeMapping.get(EFrontType.XML.name()))));
        } else {
            Map<String, Object> configToMap = ComponentConfigUtils.convertComponentConfigToMap(configs);
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configs)));
            componentVO.setComponentConfig(JSONObject.toJSONString(configToMap));
            componentVO.setDeployType(component.getDeployType());
        }
        if (isConvertHadoopVersion && isHadoopControl) {
            // 设置hadoopVersion 的key 如cdh 5.1.x
            ComponentConfig componentConfig = componentConfigMapper.listByKey(component.getId(), ConfigConstant.HADOOP_VERSION);
            if (null != componentConfig) {
                componentVO.setVersionValue(componentConfig.getValue());
            } else if (StringUtils.isNotBlank(component.getVersionValue())) {
                // 兼容老数据
                String dependName = "hadoop3".equalsIgnoreCase(component.getVersionValue()) || component.getVersionValue().startsWith("3") ? "Hadoop3" : "Hadoop2";
                List<Dict> hadoopVersion = dictMapper.getByDependName(DictType.HADOOP_VERSION.type, dependName);
                if (!CollectionUtils.isEmpty(hadoopVersion)) {
                    componentVO.setVersionValue(hadoopVersion.get(0).getDictName());
                }
            }
        }
        // 多版本才需要调用
        if (customComponent.multiVersion()) {
            customComponent.addComponent(componentVO);
        }
    }
    componentVoList.addAll(componentVoMap.values());
    return componentVoList;
}

Example 3 with IComponentVO

use of com.dtstack.taier.scheduler.vo.IComponentVO in project Taier by DTStack.

the class ConsoleComponentService method addOrUpdateComponent.

@Transactional(rollbackFor = Exception.class)
public ComponentVO addOrUpdateComponent(Long clusterId, String componentConfig, List<Resource> resources, String versionName, String kerberosFileName, String componentTemplate, EComponentType componentType, Integer storeType, String principals, String principal, boolean isMetadata, Boolean isDefault, Integer deployType) {
    if (StringUtils.isBlank(componentConfig)) {
        componentConfig = new JSONObject().toJSONString();
    }
    EComponentType storeComponent = null == storeType ? null : EComponentType.getByCode(storeType);
    PartCluster partCluster = clusterFactory.newImmediatelyLoadCluster(clusterId);
    Part part = partCluster.create(componentType, versionName, storeComponent, deployType);
    String versionValue = part.getVersionValue();
    String pluginName = part.getPluginName();
    Component componentDTO = new Component();
    componentDTO.setComponentTypeCode(componentType.getTypeCode());
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    String clusterName = cluster.getClusterName();
    Component addComponent = new Component();
    BeanUtils.copyProperties(componentDTO, addComponent);
    // 判断是否是更新组件, 需要校验组件版本
    com.dtstack.taier.dao.domain.Component dbComponent = componentMapper.getByClusterIdAndComponentType(clusterId, componentType.getTypeCode(), ComponentVersionUtil.isMultiVersionComponent(componentType.getTypeCode()) ? versionValue : null, deployType);
    String dbHadoopVersion = "";
    boolean isUpdate = false;
    boolean isOpenKerberos = isOpenKerberos(kerberosFileName, dbComponent);
    if (null != dbComponent) {
        // 更新
        dbHadoopVersion = dbComponent.getVersionValue();
        addComponent = dbComponent;
        isUpdate = true;
    }
    EComponentType storesComponent = this.checkStoresComponent(clusterId, storeType);
    addComponent.setStoreType(storesComponent.getTypeCode());
    addComponent.setVersionValue(versionValue);
    addComponent.setComponentName(componentType.getName());
    addComponent.setComponentTypeCode(componentType.getTypeCode());
    addComponent.setDeployType(deployType);
    if (EComponentType.HDFS == componentType) {
        // hdfs的组件和yarn组件的版本保持强一致
        com.dtstack.taier.dao.domain.Component yarnComponent = componentMapper.getByClusterIdAndComponentType(clusterId, EComponentType.YARN.getTypeCode(), null, null);
        if (null != yarnComponent) {
            versionName = yarnComponent.getVersionName();
        }
    }
    addComponent.setVersionName(versionName);
    if (StringUtils.isNotBlank(kerberosFileName)) {
        addComponent.setKerberosFileName(kerberosFileName);
    }
    changeDefault(BooleanUtils.isTrue(isDefault), clusterId, componentType, addComponent);
    // md5zip 会作为 config 表的一个属性
    String md5Key = updateResource(clusterId, componentConfig, resources, kerberosFileName, componentType.getTypeCode(), principals, principal, addComponent, dbComponent);
    addComponent.setClusterId(clusterId);
    if (isUpdate) {
        componentMapper.updateById(addComponent);
        refreshVersion(componentType, clusterId, addComponent, dbHadoopVersion);
        clusterMapper.updateGmtModified(clusterId);
    } else {
        componentMapper.insert(addComponent);
    }
    changeMetadata(componentType.getTypeCode(), isMetadata, clusterId, addComponent.getIsMetadata());
    List<ClientTemplate> clientTemplates = this.buildTemplate(componentType, componentConfig, isOpenKerberos, md5Key, componentTemplate, pluginName);
    componentConfigService.addOrUpdateComponentConfig(clientTemplates, addComponent.getId(), addComponent.getClusterId(), componentType.getTypeCode());
    // 此时不需要查询默认版本
    List<IComponentVO> componentVos = componentConfigService.getComponentVoByComponent(Lists.newArrayList(addComponent), true, clusterId, true, false);
    this.updateCache();
    if (CollectionUtils.isNotEmpty(componentVos)) {
        ComponentVO componentVO = (ComponentVO) componentVos.get(0);
        componentVO.setClusterName(clusterName);
        componentVO.setPrincipal(principal);
        componentVO.setPrincipals(principals);
        componentVO.setDeployType(deployType);
        componentVO.setIsMetadata(BooleanUtils.toInteger(isMetadata));
        return componentVO;
    }
    return null;
}

Example 4 with IComponentVO

use of com.dtstack.taier.scheduler.vo.IComponentVO in project Taier by DTStack.

the class ConsoleClusterService method convertComponentToScheduling.

private List<SchedulingVo> convertComponentToScheduling(Table<Integer, String, KerberosConfig> kerberosTable, Map<EComponentScheduleType, List<IComponentVO>> scheduleType) {
    List<SchedulingVo> schedulingVos = new ArrayList<>();
    // 为空也返回
    for (EComponentScheduleType value : EComponentScheduleType.values()) {
        SchedulingVo schedulingVo = new SchedulingVo();
        schedulingVo.setSchedulingCode(value.getType());
        schedulingVo.setSchedulingName(value.getName());
        List<IComponentVO> componentVoList = scheduleType.getOrDefault(value, Collections.emptyList());
        if (Objects.nonNull(kerberosTable) && !kerberosTable.isEmpty() && CollectionUtils.isNotEmpty(componentVoList)) {
            componentVoList.forEach(component -> {
                // 组件每个版本设置k8s参数
                for (ComponentVO componentVO : component.loadComponents()) {
                    KerberosConfig kerberosConfig;
                    EComponentType type = EComponentType.getByCode(componentVO.getComponentTypeCode());
                    if (type == EComponentType.YARN || type == EComponentType.SPARK_THRIFT || type == EComponentType.HIVE_SERVER) {
                        kerberosConfig = kerberosTable.get(type.getTypeCode(), StringUtils.EMPTY);
                    } else {
                        kerberosConfig = kerberosTable.get(componentVO.getComponentTypeCode(), StringUtils.isBlank(componentVO.getVersionValue()) ? StringUtils.EMPTY : componentVO.getVersionValue());
                    }
                    if (Objects.nonNull(kerberosConfig)) {
                        componentVO.setPrincipal(kerberosConfig.getPrincipal());
                        componentVO.setPrincipals(kerberosConfig.getPrincipals());
                        componentVO.setMergeKrb5Content(kerberosConfig.getMergeKrbContent());
                    }
                }
            });
        }
        schedulingVo.setComponents(componentVoList);
        schedulingVos.add(schedulingVo);
    }
    return schedulingVos;
}