Examples with Component com.dtstack.taier.dao.domain.Component used on opensource projects

Example 6 with Component

use of com.dtstack.taier.dao.domain.Component in project Taier by DTStack.

the class BatchTaskService method getComponentVersionByTaskType.

/**
 * 获取组件版本
 * @param tenantId
 * @param taskType
 * @return
 */
public List<BatchTaskGetComponentVersionResultVO> getComponentVersionByTaskType(Long tenantId, Integer taskType) {
    List<Component> components = componentService.getComponentVersionByEngineType(tenantId, taskType);
    List<BatchTaskGetComponentVersionResultVO> componentVersionResultVOS = Lists.newArrayList();
    for (Component component : components) {
        BatchTaskGetComponentVersionResultVO resultVO = new BatchTaskGetComponentVersionResultVO();
        resultVO.setComponentVersion(component.getVersionValue());
        resultVO.setDefault(component.getIsDefault());
        componentVersionResultVOS.add(resultVO);
    }
    componentVersionResultVOS.sort(sortComponentVersion());
    return componentVersionResultVOS;
}

Example 7 with Component

use of com.dtstack.taier.dao.domain.Component in project Taier by DTStack.

the class SyncOperatorPipeline method replaceTablePath.

/**
 * 创建脏数据表的分区数据
 *
 * @param saveDirty
 * @param sqlText
 * @param taskName
 * @param sourceType
 * @return
 * @throws Exception
 */
public String replaceTablePath(boolean saveDirty, String sqlText, String taskName, Integer sourceType, String db, Long tenantId) throws Exception {
    if (StringUtils.isBlank(db)) {
        return sqlText;
    }
    JSONObject sqlObject = JSONObject.parseObject(sqlText);
    JSONObject job = sqlObject.getJSONObject("job");
    JSONObject setting = job.getJSONObject("setting");
    if (setting.containsKey("dirty")) {
        if (!saveDirty) {
            setting.remove("dirty");
            return sqlObject.toJSONString();
        }
        JSONObject dirty = setting.getJSONObject("dirty");
        String tableName = dirty.getString("tableName");
        String path = null;
        if (StringUtils.isNotEmpty(tableName)) {
            // 任务提交到task 之前 脏数据表 必须要在 ide 创建
            if (!tableName.contains(".")) {
                tableName = String.format("%s.%s", db, tableName);
            }
            Long time = Timestamp.valueOf(LocalDateTime.now()).getTime();
            String alterSql = String.format(ADD_PART_TEMP, tableName, taskName, time);
            String location = "";
            if (DataSourceType.hadoopDirtyDataSource.contains(sourceType)) {
                Cluster cluster = clusterService.getCluster(tenantId);
                Component metadataComponent = componentService.getMetadataComponent(cluster.getId());
                EComponentType metadataComponentType = EComponentType.getByCode(null == metadataComponent ? EComponentType.SPARK_THRIFT.getTypeCode() : metadataComponent.getComponentTypeCode());
                JSONObject pluginInfo = clusterService.getConfigByKey(tenantId, metadataComponentType.getConfName(), null);
                String typeName = getHiveTypeName(DataSourceType.getSourceType(sourceType));
                pluginInfo.put(ConfigConstant.TYPE_NAME_KEY, typeName);
                pluginInfo.compute(ConfigConstant.JDBCURL, (jdbcUrl, val) -> {
                    String jdbcUrlVal = (String) val;
                    if (StringUtils.isBlank(jdbcUrlVal)) {
                        return null;
                    }
                    return jdbcUrlVal.replace("/%s", environmentContext.getComponentJdbcToReplace());
                });
                workerOperator.executeQuery(pluginInfo.toJSONString(), alterSql, db);
                location = this.getTableLocation(pluginInfo, db, String.format("desc formatted %s", tableName));
            }
            if (StringUtils.isBlank(location)) {
                LOGGER.warn("table {} replace dirty path is null,dirtyType {} ", tableName, sourceType);
            }
            String partName = String.format("task_name=%s/time=%s", taskName, time);
            path = location + "/" + partName;
            dirty.put("path", path);
            setting.put("dirty", dirty);
            job.put("setting", setting);
            sqlObject.put("job", job);
        }
    }
    return sqlObject.toJSONString();
}

Example 8 with Component

use of com.dtstack.taier.dao.domain.Component in project Taier by DTStack.

the class ClusterService method getConfigByKeyByClusterId.

public JSONObject getConfigByKeyByClusterId(Long clusterId, String componentConfName, String componentVersion) {
    // 根据组件区分kerberos
    EComponentType componentType = EComponentType.getByConfName(componentConfName);
    Component component = componentMapper.getByClusterIdAndComponentType(clusterId, componentType.getTypeCode(), componentVersion, null);
    if (null == component) {
        return null;
    }
    JSONObject configObj = componentService.getComponentByClusterId(clusterId, component.getComponentTypeCode(), false, JSONObject.class, componentVersion);
    if (configObj != null) {
        if (StringUtils.isNotBlank(component.getKerberosFileName())) {
            // 开启kerberos的kerberosFileName不为空
            KerberosConfig kerberosConfig = consoleKerberosMapper.getByComponentType(clusterId, componentType.getTypeCode(), ComponentVersionUtil.isMultiVersionComponent(componentType.getTypeCode()) ? StringUtils.isNotBlank(componentVersion) ? componentVersion : componentMapper.getDefaultComponentVersionByClusterAndComponentType(clusterId, componentType.getTypeCode()) : null);
            // 添加组件的kerberos配置信息 应用层使用
            configObj.put(ConfigConstant.KERBEROS_CONFIG, kerberosConfig);
            // 填充sftp配置项
            Map sftpMap = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, Map.class, null);
            if (MapUtils.isNotEmpty(sftpMap)) {
                configObj.put(EComponentType.SFTP.getConfName(), sftpMap);
            }
        }
        // 返回版本
        configObj.put(ConfigConstant.VERSION, component.getVersionValue());
        configObj.put(IS_METADATA, component.getIsMetadata());
        return configObj;
    }
    return null;
}

Example 9 with Component

use of com.dtstack.taier.dao.domain.Component in project Taier by DTStack.

the class ComponentConfigService method getComponentVoByComponent.

public List<IComponentVO> getComponentVoByComponent(List<Component> components, boolean isFilter, Long clusterId, boolean isConvertHadoopVersion, boolean multiVersion) {
    if (null == clusterId) {
        throw new RdosDefineException("集群id不能为空");
    }
    if (CollectionUtils.isEmpty(components)) {
        return new ArrayList<>(0);
    }
    // 集群所关联的组件的配置
    List<ComponentConfig> componentConfigs = componentConfigMapper.listByClusterId(clusterId, isFilter);
    if (CollectionUtils.isEmpty(componentConfigs)) {
        return new ArrayList<>(0);
    }
    // 组件按类型分组, 因为可能存在组件有多个版本, 此时需要兼容单版本和多版本格式问题
    Map<Integer, IComponentVO> componentVoMap = new HashMap<>(components.size());
    components.stream().collect(Collectors.groupingBy(Component::getComponentTypeCode, Collectors.toList())).forEach((k, v) -> componentVoMap.put(k, multiVersion ? ComponentMultiVersionVO.getInstanceWithCapacityAndType(k, v.size()) : ComponentVO.getInstance()));
    // 配置按照组件进行分组, 存在组件有多个版本
    Map<Long, List<ComponentConfig>> componentIdConfigs = componentConfigs.stream().collect(Collectors.groupingBy(ComponentConfig::getComponentId));
    List<IComponentVO> componentVoList = new ArrayList<>(components.size());
    for (Component component : components) {
        IComponentVO customComponent = componentVoMap.get(component.getComponentTypeCode());
        ComponentVO componentVO = IComponentVO.getComponentVo(customComponent, component);
        ;
        // 当前组件的配置
        List<ComponentConfig> configs = componentIdConfigs.get(component.getId());
        // hdfs yarn 才将自定义参数移除 过滤返回给前端
        boolean isHadoopControl = EComponentType.hadoopVersionComponents.contains(EComponentType.getByCode(component.getComponentTypeCode()));
        if (isHadoopControl) {
            // 配置按照编辑类型进行分组
            Map<String, List<ComponentConfig>> configTypeMapping = configs.stream().collect(Collectors.groupingBy(ComponentConfig::getType));
            // hdfs yarn 4.1 template只有自定义参数
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configTypeMapping.get(EFrontType.CUSTOM_CONTROL.name()))));
            // hdfs yarn 4.1 config为xml配置参数
            componentVO.setComponentConfig(JSONObject.toJSONString(ComponentConfigUtils.convertComponentConfigToMap(configTypeMapping.get(EFrontType.XML.name()))));
        } else {
            Map<String, Object> configToMap = ComponentConfigUtils.convertComponentConfigToMap(configs);
            componentVO.setComponentTemplate(JSONObject.toJSONString(ComponentConfigUtils.buildDBDataToClientTemplate(configs)));
            componentVO.setComponentConfig(JSONObject.toJSONString(configToMap));
            componentVO.setDeployType(component.getDeployType());
        }
        if (isConvertHadoopVersion && isHadoopControl) {
            // 设置hadoopVersion 的key 如cdh 5.1.x
            ComponentConfig componentConfig = componentConfigMapper.listByKey(component.getId(), ConfigConstant.HADOOP_VERSION);
            if (null != componentConfig) {
                componentVO.setVersionValue(componentConfig.getValue());
            } else if (StringUtils.isNotBlank(component.getVersionValue())) {
                // 兼容老数据
                String dependName = "hadoop3".equalsIgnoreCase(component.getVersionValue()) || component.getVersionValue().startsWith("3") ? "Hadoop3" : "Hadoop2";
                List<Dict> hadoopVersion = dictMapper.getByDependName(DictType.HADOOP_VERSION.type, dependName);
                if (!CollectionUtils.isEmpty(hadoopVersion)) {
                    componentVO.setVersionValue(hadoopVersion.get(0).getDictName());
                }
            }
        }
        // 多版本才需要调用
        if (customComponent.multiVersion()) {
            customComponent.addComponent(componentVO);
        }
    }
    componentVoList.addAll(componentVoMap.values());
    return componentVoList;
}

Example 10 with Component

use of com.dtstack.taier.dao.domain.Component in project Taier by DTStack.

the class ComponentService method getComponentVersionByEngineType.

public List<Component> getComponentVersionByEngineType(Long tenantId, Integer taskType) {
    EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskType);
    EComponentType componentType = scheduleJobType.getComponentType();
    List<Component> componentVersionList = componentMapper.getComponentVersionByEngineType(tenantId, componentType.getTypeCode());
    if (CollectionUtils.isEmpty(componentVersionList)) {
        return Collections.emptyList();
    }
    Set<String> distinct = new HashSet<>(2);
    List<Component> components = new ArrayList<>(2);
    for (Component component : componentVersionList) {
        if (distinct.add(component.getVersionValue())) {
            components.add(component);
        }
    }
    return components;
}