Examples with EComponentType com.dtstack.taier.common.enums.EComponentType used on opensource projects

Example 11 with EComponentType

use of com.dtstack.taier.common.enums.EComponentType in project Taier by DTStack.

the class SyncOperatorPipeline method replaceTablePath.

/**
 * 创建脏数据表的分区数据
 *
 * @param saveDirty
 * @param sqlText
 * @param taskName
 * @param sourceType
 * @return
 * @throws Exception
 */
public String replaceTablePath(boolean saveDirty, String sqlText, String taskName, Integer sourceType, String db, Long tenantId) throws Exception {
    if (StringUtils.isBlank(db)) {
        return sqlText;
    }
    JSONObject sqlObject = JSONObject.parseObject(sqlText);
    JSONObject job = sqlObject.getJSONObject("job");
    JSONObject setting = job.getJSONObject("setting");
    if (setting.containsKey("dirty")) {
        if (!saveDirty) {
            setting.remove("dirty");
            return sqlObject.toJSONString();
        }
        JSONObject dirty = setting.getJSONObject("dirty");
        String tableName = dirty.getString("tableName");
        String path = null;
        if (StringUtils.isNotEmpty(tableName)) {
            // 任务提交到task 之前 脏数据表 必须要在 ide 创建
            if (!tableName.contains(".")) {
                tableName = String.format("%s.%s", db, tableName);
            }
            Long time = Timestamp.valueOf(LocalDateTime.now()).getTime();
            String alterSql = String.format(ADD_PART_TEMP, tableName, taskName, time);
            String location = "";
            if (DataSourceType.hadoopDirtyDataSource.contains(sourceType)) {
                Cluster cluster = clusterService.getCluster(tenantId);
                Component metadataComponent = componentService.getMetadataComponent(cluster.getId());
                EComponentType metadataComponentType = EComponentType.getByCode(null == metadataComponent ? EComponentType.SPARK_THRIFT.getTypeCode() : metadataComponent.getComponentTypeCode());
                JSONObject pluginInfo = clusterService.getConfigByKey(tenantId, metadataComponentType.getConfName(), null);
                String typeName = getHiveTypeName(DataSourceType.getSourceType(sourceType));
                pluginInfo.put(ConfigConstant.TYPE_NAME_KEY, typeName);
                pluginInfo.compute(ConfigConstant.JDBCURL, (jdbcUrl, val) -> {
                    String jdbcUrlVal = (String) val;
                    if (StringUtils.isBlank(jdbcUrlVal)) {
                        return null;
                    }
                    return jdbcUrlVal.replace("/%s", environmentContext.getComponentJdbcToReplace());
                });
                workerOperator.executeQuery(pluginInfo.toJSONString(), alterSql, db);
                location = this.getTableLocation(pluginInfo, db, String.format("desc formatted %s", tableName));
            }
            if (StringUtils.isBlank(location)) {
                LOGGER.warn("table {} replace dirty path is null,dirtyType {} ", tableName, sourceType);
            }
            String partName = String.format("task_name=%s/time=%s", taskName, time);
            path = location + "/" + partName;
            dirty.put("path", path);
            setting.put("dirty", dirty);
            job.put("setting", setting);
            sqlObject.put("job", job);
        }
    }
    return sqlObject.toJSONString();
}

Example 12 with EComponentType

use of com.dtstack.taier.common.enums.EComponentType in project Taier by DTStack.

the class ClusterService method getConfigByKeyByClusterId.

public JSONObject getConfigByKeyByClusterId(Long clusterId, String componentConfName, String componentVersion) {
    // 根据组件区分kerberos
    EComponentType componentType = EComponentType.getByConfName(componentConfName);
    Component component = componentMapper.getByClusterIdAndComponentType(clusterId, componentType.getTypeCode(), componentVersion, null);
    if (null == component) {
        return null;
    }
    JSONObject configObj = componentService.getComponentByClusterId(clusterId, component.getComponentTypeCode(), false, JSONObject.class, componentVersion);
    if (configObj != null) {
        if (StringUtils.isNotBlank(component.getKerberosFileName())) {
            // 开启kerberos的kerberosFileName不为空
            KerberosConfig kerberosConfig = consoleKerberosMapper.getByComponentType(clusterId, componentType.getTypeCode(), ComponentVersionUtil.isMultiVersionComponent(componentType.getTypeCode()) ? StringUtils.isNotBlank(componentVersion) ? componentVersion : componentMapper.getDefaultComponentVersionByClusterAndComponentType(clusterId, componentType.getTypeCode()) : null);
            // 添加组件的kerberos配置信息 应用层使用
            configObj.put(ConfigConstant.KERBEROS_CONFIG, kerberosConfig);
            // 填充sftp配置项
            Map sftpMap = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, Map.class, null);
            if (MapUtils.isNotEmpty(sftpMap)) {
                configObj.put(EComponentType.SFTP.getConfName(), sftpMap);
            }
        }
        // 返回版本
        configObj.put(ConfigConstant.VERSION, component.getVersionValue());
        configObj.put(IS_METADATA, component.getIsMetadata());
        return configObj;
    }
    return null;
}

Example 13 with EComponentType

use of com.dtstack.taier.common.enums.EComponentType in project Taier by DTStack.

the class ComponentService method getComponentVersionByEngineType.

public List<Component> getComponentVersionByEngineType(Long tenantId, Integer taskType) {
    EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskType);
    EComponentType componentType = scheduleJobType.getComponentType();
    List<Component> componentVersionList = componentMapper.getComponentVersionByEngineType(tenantId, componentType.getTypeCode());
    if (CollectionUtils.isEmpty(componentVersionList)) {
        return Collections.emptyList();
    }
    Set<String> distinct = new HashSet<>(2);
    List<Component> components = new ArrayList<>(2);
    for (Component component : componentVersionList) {
        if (distinct.add(component.getVersionValue())) {
            components.add(component);
        }
    }
    return components;
}

Example 14 with EComponentType

use of com.dtstack.taier.common.enums.EComponentType in project Taier by DTStack.

the class ScheduleDictService method convertVersionNameToValue.

public String convertVersionNameToValue(String componentVersion, Integer taskType) {
    if (StringUtils.isBlank(componentVersion)) {
        return "";
    }
    EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskType);
    EComponentType componentType = scheduleJobType.getComponentType();
    Integer dictType = DictType.getByEComponentType(componentType);
    if (null != dictType) {
        Dict versionDict = getByNameAndValue(dictType, componentVersion.trim(), null, null);
        if (null != versionDict) {
            return versionDict.getDictValue();
        }
    }
    return "";
}

Example 15 with EComponentType

use of com.dtstack.taier.common.enums.EComponentType in project Taier by DTStack.

the class BatchComponentService method getAllDataBases.

/**
 * 获取集群对应组件下数据源中的databases
 *
 * @param clusterId         集群ID
 * @param componentTypeCode 组件类型
 * @return
 */
public List<String> getAllDataBases(Long clusterId, Integer componentTypeCode) {
    EComponentType eComponentType = EComponentType.getByCode(componentTypeCode);
    IComponentService componentService = multiEngineServiceFactory.getComponentService(componentTypeCode);
    return componentService.getAllDataBases(clusterId, eComponentType, "");
}