Examples with Cluster com.dtstack.taier.dao.domain.Cluster used on opensource projects

Example 1 with Cluster

use of com.dtstack.taier.dao.domain.Cluster in project Taier by DTStack.

the class ConsoleClusterService method getConsoleClusterInfo.

/**
 * 获取集群信息详情 需要根据组件分组
 *
 * @param clusterId
 * @return
 */
public ClusterVO getConsoleClusterInfo(Long clusterId) {
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        return new ClusterVO();
    }
    ClusterVO clusterVO = ClusterVO.toVO(cluster);
    // 查询默认版本或者多个版本
    List<com.dtstack.taier.dao.domain.Component> components = componentMapper.listByClusterId(clusterId, null, false);
    List<IComponentVO> componentConfigs = componentConfigService.getComponentVoByComponent(components, true, clusterId, true, true);
    Table<Integer, String, KerberosConfig> kerberosTable = null;
    // kerberos的配置
    kerberosTable = HashBasedTable.create();
    for (KerberosConfig kerberosConfig : consoleKerberosMapper.getByClusters(clusterId)) {
        kerberosTable.put(kerberosConfig.getComponentType(), StringUtils.isBlank(kerberosConfig.getComponentVersion()) ? StringUtils.EMPTY : kerberosConfig.getComponentVersion(), kerberosConfig);
    }
    Map<EComponentScheduleType, List<IComponentVO>> scheduleType = new HashMap<>(4);
    // 组件根据用途分组(计算,资源)
    if (CollectionUtils.isNotEmpty(componentConfigs)) {
        scheduleType = componentConfigs.stream().collect(Collectors.groupingBy(c -> EComponentType.getScheduleTypeByComponent(c.getComponentTypeCode())));
    }
    List<SchedulingVo> schedulingVos = convertComponentToScheduling(kerberosTable, scheduleType);
    clusterVO.setScheduling(schedulingVos);
    clusterVO.setCanModifyMetadata(checkMetadata(clusterId, components));
    return clusterVO;
}

Example 2 with Cluster

use of com.dtstack.taier.dao.domain.Cluster in project Taier by DTStack.

the class TenantController method pageQuery.

@PostMapping(value = "/pageQuery")
public R<PageResult<List<ClusterTenantVO>>> pageQuery(@RequestParam("clusterId") Long clusterId, @RequestParam("name") String tenantName, @RequestParam("pageSize") int pageSize, @RequestParam("currentPage") int currentPage) {
    Cluster cluster = clusterService.getCluster(clusterId);
    if (cluster == null) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    tenantName = tenantName == null ? "" : tenantName;
    return R.ok(tenantService.pageQuery(clusterId, tenantName, pageSize, currentPage));
}

Example 3 with Cluster

use of com.dtstack.taier.dao.domain.Cluster in project Taier by DTStack.

the class TenantController method bindingTenant.

@PostMapping(value = "/bindingTenant")
public R<Void> bindingTenant(@RequestBody ComponentBindTenantVO vo) throws Exception {
    Cluster cluster = clusterService.getCluster(vo.getClusterId());
    if (cluster == null) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    if (CollectionUtils.isNotEmpty(vo.getBindDBList())) {
        for (ComponentBindDBVO bindDBVO : vo.getBindDBList()) {
            if (Objects.isNull(bindDBVO.getComponentCode())) {
                throw new RdosDefineException(String.format(ErrorCode.META_COMPONENT_NOT_EXISTS.getMsg(), EComponentType.SPARK_THRIFT.getName()));
            }
        }
    }
    tenantService.bindingTenant(vo.getTenantId(), vo.getClusterId(), vo.getQueueId(), cluster.getClusterName(), vo.getBindDBList());
    return R.empty();
}

Example 4 with Cluster

use of com.dtstack.taier.dao.domain.Cluster in project Taier by DTStack.

the class SyncOperatorPipeline method replaceTablePath.

/**
 * 创建脏数据表的分区数据
 *
 * @param saveDirty
 * @param sqlText
 * @param taskName
 * @param sourceType
 * @return
 * @throws Exception
 */
public String replaceTablePath(boolean saveDirty, String sqlText, String taskName, Integer sourceType, String db, Long tenantId) throws Exception {
    if (StringUtils.isBlank(db)) {
        return sqlText;
    }
    JSONObject sqlObject = JSONObject.parseObject(sqlText);
    JSONObject job = sqlObject.getJSONObject("job");
    JSONObject setting = job.getJSONObject("setting");
    if (setting.containsKey("dirty")) {
        if (!saveDirty) {
            setting.remove("dirty");
            return sqlObject.toJSONString();
        }
        JSONObject dirty = setting.getJSONObject("dirty");
        String tableName = dirty.getString("tableName");
        String path = null;
        if (StringUtils.isNotEmpty(tableName)) {
            // 任务提交到task 之前 脏数据表 必须要在 ide 创建
            if (!tableName.contains(".")) {
                tableName = String.format("%s.%s", db, tableName);
            }
            Long time = Timestamp.valueOf(LocalDateTime.now()).getTime();
            String alterSql = String.format(ADD_PART_TEMP, tableName, taskName, time);
            String location = "";
            if (DataSourceType.hadoopDirtyDataSource.contains(sourceType)) {
                Cluster cluster = clusterService.getCluster(tenantId);
                Component metadataComponent = componentService.getMetadataComponent(cluster.getId());
                EComponentType metadataComponentType = EComponentType.getByCode(null == metadataComponent ? EComponentType.SPARK_THRIFT.getTypeCode() : metadataComponent.getComponentTypeCode());
                JSONObject pluginInfo = clusterService.getConfigByKey(tenantId, metadataComponentType.getConfName(), null);
                String typeName = getHiveTypeName(DataSourceType.getSourceType(sourceType));
                pluginInfo.put(ConfigConstant.TYPE_NAME_KEY, typeName);
                pluginInfo.compute(ConfigConstant.JDBCURL, (jdbcUrl, val) -> {
                    String jdbcUrlVal = (String) val;
                    if (StringUtils.isBlank(jdbcUrlVal)) {
                        return null;
                    }
                    return jdbcUrlVal.replace("/%s", environmentContext.getComponentJdbcToReplace());
                });
                workerOperator.executeQuery(pluginInfo.toJSONString(), alterSql, db);
                location = this.getTableLocation(pluginInfo, db, String.format("desc formatted %s", tableName));
            }
            if (StringUtils.isBlank(location)) {
                LOGGER.warn("table {} replace dirty path is null,dirtyType {} ", tableName, sourceType);
            }
            String partName = String.format("task_name=%s/time=%s", taskName, time);
            path = location + "/" + partName;
            dirty.put("path", path);
            setting.put("dirty", dirty);
            job.put("setting", setting);
            sqlObject.put("job", job);
        }
    }
    return sqlObject.toJSONString();
}

Example 5 with Cluster

use of com.dtstack.taier.dao.domain.Cluster in project Taier by DTStack.

the class ClusterService method buildClusterConfig.

public JSONObject buildClusterConfig(Long clusterId, String componentVersion, EComponentType computeComponentType) {
    Cluster cluster = clusterMapper.getOne(clusterId);
    if (null == cluster) {
        throw new RdosDefineException(ErrorCode.CANT_NOT_FIND_CLUSTER);
    }
    JSONObject config = new JSONObject();
    List<Component> components = componentService.listAllComponents(clusterId);
    for (Component component : components) {
        EComponentType componentType = EComponentType.getByCode(component.getComponentTypeCode());
        if (!EComponentScheduleType.COMPUTE.equals(EComponentType.getScheduleTypeByComponent(component.getComponentTypeCode()))) {
            JSONObject componentConfig = componentService.getComponentByClusterId(clusterId, componentType.getTypeCode(), false, JSONObject.class, null);
            config.put(componentType.getConfName(), componentConfig);
        } else if (componentType.equals(computeComponentType)) {
            JSONObject componentConfig = componentService.getComponentByClusterId(clusterId, componentType.getTypeCode(), false, JSONObject.class, componentVersion);
            config.put(componentType.getConfName(), componentConfig);
        }
    // ignore other compute component
    }
    config.put(CLUSTER, cluster.getClusterName());
    return config;
}