Examples with ClientTemplate com.dtstack.taier.scheduler.impl.pojo.ClientTemplate used on opensource projects

Example 6 with ClientTemplate

use of com.dtstack.taier.scheduler.impl.pojo.ClientTemplate in project Taier by DTStack.

the class ComponentConfigUtils method componentConfigToTemplate.

private static ClientTemplate componentConfigToTemplate(ComponentConfig componentConfig) {
    ClientTemplate clientTemplate = new ClientTemplate();
    BeanUtils.copyProperties(componentConfig, clientTemplate);
    clientTemplate.setRequired(BooleanUtils.toBoolean(componentConfig.getRequired()));
    if (componentConfig.getValue().startsWith("[")) {
        clientTemplate.setValue(JSONArray.parseArray(componentConfig.getValue()));
    } else {
        if (StringUtils.isBlank(componentConfig.getValue()) && EFrontType.GROUP.name().equalsIgnoreCase(componentConfig.getType())) {
            // group key 和value 同值
            clientTemplate.setValue(componentConfig.getKey());
        } else {
            clientTemplate.setValue(componentConfig.getValue());
        }
    }
    if (null == clientTemplate.getDependencyValue()) {
        clientTemplate.setDependencyKey(null);
    }
    return clientTemplate;
}

Example 7 with ClientTemplate

use of com.dtstack.taier.scheduler.impl.pojo.ClientTemplate in project Taier by DTStack.

the class ComponentConfigUtils method convertXMLConfigToComponentConfig.

/**
 * 将yarn hdfs 等xml配置信息转换为clientTemplate
 *
 * @param componentConfigString
 * @return
 */
public static List<ClientTemplate> convertXMLConfigToComponentConfig(String componentConfigString) {
    if (StringUtils.isBlank(componentConfigString)) {
        return new ArrayList<>(0);
    }
    JSONObject componentConfigObj = JSONObject.parseObject(componentConfigString);
    List<ClientTemplate> configs = new ArrayList<>(componentConfigObj.size());
    for (String key : componentConfigObj.keySet()) {
        configs.add(buildCustom(key, componentConfigObj.get(key), EFrontType.XML.name()));
    }
    return configs;
}

Example 8 with ClientTemplate

use of com.dtstack.taier.scheduler.impl.pojo.ClientTemplate in project Taier by DTStack.

the class ConsoleComponentService method addOrUpdateComponent.

@Transactional(rollbackFor = Exception.class)
public ComponentVO addOrUpdateComponent(Long clusterId, String componentConfig, List<Resource> resources, String versionName, String kerberosFileName, String componentTemplate, EComponentType componentType, Integer storeType, String principals, String principal, boolean isMetadata, Boolean isDefault, Integer deployType) {
    if (StringUtils.isBlank(componentConfig)) {
        componentConfig = new JSONObject().toJSONString();
    }
    EComponentType storeComponent = null == storeType ? null : EComponentType.getByCode(storeType);
    PartCluster partCluster = clusterFactory.newImmediatelyLoadCluster(clusterId);
    Part part = partCluster.create(componentType, versionName, storeComponent, deployType);
    String versionValue = part.getVersionValue();
    String pluginName = part.getPluginName();
    Component componentDTO = new Component();
    componentDTO.setComponentTypeCode(componentType.getTypeCode());
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    String clusterName = cluster.getClusterName();
    Component addComponent = new Component();
    BeanUtils.copyProperties(componentDTO, addComponent);
    // 判断是否是更新组件, 需要校验组件版本
    com.dtstack.taier.dao.domain.Component dbComponent = componentMapper.getByClusterIdAndComponentType(clusterId, componentType.getTypeCode(), ComponentVersionUtil.isMultiVersionComponent(componentType.getTypeCode()) ? versionValue : null, deployType);
    String dbHadoopVersion = "";
    boolean isUpdate = false;
    boolean isOpenKerberos = isOpenKerberos(kerberosFileName, dbComponent);
    if (null != dbComponent) {
        // 更新
        dbHadoopVersion = dbComponent.getVersionValue();
        addComponent = dbComponent;
        isUpdate = true;
    }
    EComponentType storesComponent = this.checkStoresComponent(clusterId, storeType);
    addComponent.setStoreType(storesComponent.getTypeCode());
    addComponent.setVersionValue(versionValue);
    addComponent.setComponentName(componentType.getName());
    addComponent.setComponentTypeCode(componentType.getTypeCode());
    addComponent.setDeployType(deployType);
    if (EComponentType.HDFS == componentType) {
        // hdfs的组件和yarn组件的版本保持强一致
        com.dtstack.taier.dao.domain.Component yarnComponent = componentMapper.getByClusterIdAndComponentType(clusterId, EComponentType.YARN.getTypeCode(), null, null);
        if (null != yarnComponent) {
            versionName = yarnComponent.getVersionName();
        }
    }
    addComponent.setVersionName(versionName);
    if (StringUtils.isNotBlank(kerberosFileName)) {
        addComponent.setKerberosFileName(kerberosFileName);
    }
    changeDefault(BooleanUtils.isTrue(isDefault), clusterId, componentType, addComponent);
    // md5zip 会作为 config 表的一个属性
    String md5Key = updateResource(clusterId, componentConfig, resources, kerberosFileName, componentType.getTypeCode(), principals, principal, addComponent, dbComponent);
    addComponent.setClusterId(clusterId);
    if (isUpdate) {
        componentMapper.updateById(addComponent);
        refreshVersion(componentType, clusterId, addComponent, dbHadoopVersion);
        clusterMapper.updateGmtModified(clusterId);
    } else {
        componentMapper.insert(addComponent);
    }
    changeMetadata(componentType.getTypeCode(), isMetadata, clusterId, addComponent.getIsMetadata());
    List<ClientTemplate> clientTemplates = this.buildTemplate(componentType, componentConfig, isOpenKerberos, md5Key, componentTemplate, pluginName);
    componentConfigService.addOrUpdateComponentConfig(clientTemplates, addComponent.getId(), addComponent.getClusterId(), componentType.getTypeCode());
    // 此时不需要查询默认版本
    List<IComponentVO> componentVos = componentConfigService.getComponentVoByComponent(Lists.newArrayList(addComponent), true, clusterId, true, false);
    this.updateCache();
    if (CollectionUtils.isNotEmpty(componentVos)) {
        ComponentVO componentVO = (ComponentVO) componentVos.get(0);
        componentVO.setClusterName(clusterName);
        componentVO.setPrincipal(principal);
        componentVO.setPrincipals(principals);
        componentVO.setDeployType(deployType);
        componentVO.setIsMetadata(BooleanUtils.toInteger(isMetadata));
        return componentVO;
    }
    return null;
}

Example 9 with ClientTemplate

use of com.dtstack.taier.scheduler.impl.pojo.ClientTemplate in project Taier by DTStack.

the class ConsoleComponentService method buildTemplate.

private List<ClientTemplate> buildTemplate(EComponentType componentType, String componentString, boolean isOpenKerberos, String md5Key, String clientTemplates, String pluginName) {
    List<ClientTemplate> templates = new ArrayList<>();
    JSONObject componentConfigJSON = JSONObject.parseObject(componentString);
    if (isOpenKerberos) {
        if (EComponentType.HDFS.equals(componentType)) {
            componentConfigJSON.put("dfs.namenode.kerberos.principal.pattern", "*");
        }
        if (EComponentType.YARN.equals(componentType)) {
            componentConfigJSON.put("yarn.resourcemanager.principal.pattern", "*");
        }
    }
    if (EComponentType.typeComponentVersion.contains(componentType)) {
        // 添加typeName
        ClientTemplate typeNameClientTemplate = ComponentConfigUtils.buildOthers(TYPE_NAME_KEY, pluginName);
        templates.add(typeNameClientTemplate);
    }
    if (!StringUtils.isBlank(md5Key)) {
        ClientTemplate md5ClientTemplate = ComponentConfigUtils.buildOthers(MD5_SUM_KEY, md5Key);
        templates.add(md5ClientTemplate);
    }
    if (EComponentType.noControlComponents.contains(componentType)) {
        // xml配置文件也转换为组件
        List<ClientTemplate> xmlTemplates = ComponentConfigUtils.convertXMLConfigToComponentConfig(componentConfigJSON.toJSONString());
        templates.addAll(xmlTemplates);
        // yarn 和hdfs的自定义参数
        templates.addAll(ComponentConfigUtils.dealXmlCustomControl(componentType, clientTemplates));
    } else {
        List<ClientTemplate> controlTemplate = JSONObject.parseArray(clientTemplates, ClientTemplate.class);
        templates.addAll(controlTemplate);
    }
    return templates;
}

Example 10 with ClientTemplate

use of com.dtstack.taier.scheduler.impl.pojo.ClientTemplate in project Taier by DTStack.

the class ComponentConfigUtils method saveTreeToList.

/**
 * 将展示的树结构转换为List存入数据库中
 *
 * @param reduceTemplate
 * @param clusterId
 * @param componentId
 * @param dependKey
 * @param dependValue
 */
public static List<ComponentConfig> saveTreeToList(List<ClientTemplate> reduceTemplate, Long clusterId, Long componentId, String dependKey, String dependValue, Integer componentTypeCode) {
    List<ComponentConfig> saveComponentConfigs = new ArrayList<>();
    for (ClientTemplate clientTemplate : reduceTemplate) {
        ComponentConfig componentConfig = ComponentConfigUtils.convertClientTemplateToConfig(clientTemplate);
        componentConfig.setClusterId(clusterId);
        componentConfig.setComponentId(componentId);
        if (StringUtils.isNotBlank(dependKey)) {
            componentConfig.setDependencyKey(dependKey);
        }
        if (StringUtils.isNotBlank(dependValue) && componentConfig.getType().equalsIgnoreCase(EFrontType.GROUP.name())) {
            componentConfig.setDependencyValue(dependValue);
        }
        componentConfig.setRequired(BooleanUtils.toInteger(clientTemplate.isRequired(), 1, 0, 0));
        componentConfig.setComponentTypeCode(componentTypeCode);
        saveComponentConfigs.add(componentConfig);
        if (!CollectionUtils.isEmpty(clientTemplate.getValues())) {
            String dependKeys = "";
            // deploymode$perjob$metrics.reporter.promgateway.deleteOnShutdown
            if (StringUtils.isNotBlank(dependKey)) {
                dependKeys = dependKey + dependencySeparator + clientTemplate.getKey();
            } else {
                dependKeys = clientTemplate.getKey();
            }
            List<ComponentConfig> componentConfigs = saveTreeToList(clientTemplate.getValues(), clusterId, componentId, dependKeys, clientTemplate.getDependencyValue(), componentTypeCode);
            if (!CollectionUtils.isEmpty(componentConfigs)) {
                saveComponentConfigs.addAll(componentConfigs);
            }
        }
    }
    return saveComponentConfigs;
}