Search in sources :

Example 1 with ComponentTestResult

use of com.dtstack.taier.pluginapi.pojo.ComponentTestResult in project Taier by DTStack.

the class ConsoleComponentService method refresh.

/**
 * 刷新组件信息
 *
 * @param clusterName
 * @return
 */
public List<ComponentTestResult> refresh(String clusterName) {
    List<ComponentTestResult> refreshResults = new ArrayList<>();
    ComponentTestResult componentTestResult = testConnect(clusterName, EComponentType.YARN.getTypeCode(), null);
    refreshResults.add(componentTestResult);
    return refreshResults;
}
Also used : ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult)

Example 2 with ComponentTestResult

use of com.dtstack.taier.pluginapi.pojo.ComponentTestResult in project Taier by DTStack.

the class ConsoleComponentService method testConnects.

/**
 * 测试所有组件连通性
 *
 * @param clusterName
 * @return
 */
public List<ComponentMultiTestResult> testConnects(String clusterName) {
    Cluster cluster = clusterMapper.getByClusterName(clusterName);
    List<Component> components = getComponents(cluster);
    if (CollectionUtils.isEmpty(components)) {
        return new ArrayList<>();
    }
    Map sftpMap = componentService.getComponentByClusterId(cluster.getId(), EComponentType.SFTP.getTypeCode(), false, Map.class, null);
    Map<Component, CompletableFuture<ComponentTestResult>> completableFutureMap = components.stream().collect(Collectors.toMap(component -> component, c -> CompletableFuture.supplyAsync(() -> testComponentWithResult(clusterName, cluster, sftpMap, c), connectPool)));
    CompletableFuture<List<ComponentTestResult>> completableFuture = CompletableFuture.allOf(completableFutureMap.values().toArray(new CompletableFuture[0])).thenApply((f) -> completableFutureMap.keySet().stream().map(component -> {
        try {
            return completableFutureMap.get(component).get(env.getTestConnectTimeout(), TimeUnit.SECONDS);
        } catch (Exception e) {
            ComponentTestResult testResult = new ComponentTestResult();
            testResult.setResult(false);
            testResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
            testResult.setComponentVersion(component.getVersionValue());
            testResult.setComponentTypeCode(component.getComponentTypeCode());
            return testResult;
        }
    }).collect(Collectors.toList()));
    try {
        List<ComponentTestResult> componentTestResults = completableFuture.get();
        Map<Integer, List<ComponentTestResult>> componentCodeResultMap = componentTestResults.stream().collect(Collectors.groupingBy(ComponentTestResult::getComponentTypeCode, Collectors.collectingAndThen(Collectors.toList(), c -> c)));
        return componentCodeResultMap.keySet().stream().map(componentCode -> {
            ComponentMultiTestResult multiTestResult = new ComponentMultiTestResult(componentCode);
            multiTestResult.setMultiVersion(componentCodeResultMap.get(componentCode));
            List<ComponentTestResult> testResults = componentCodeResultMap.get(componentCode);
            multiTestResult.setResult(testResults.stream().allMatch(ComponentTestResult::getResult));
            testResults.stream().filter(componentTestResult -> StringUtils.isNotBlank(componentTestResult.getErrorMsg())).findFirst().ifPresent(errorResult -> multiTestResult.setErrorMsg(errorResult.getErrorMsg()));
            return multiTestResult;
        }).collect(Collectors.toList());
    } catch (Exception e) {
        throw new RdosDefineException(e);
    }
}
Also used : IComponentVO(com.dtstack.taier.scheduler.vo.IComponentVO) ClusterTenantMapper(com.dtstack.taier.dao.mapper.ClusterTenantMapper) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) MathUtil(com.dtstack.taier.common.util.MathUtil) ComponentService(com.dtstack.taier.scheduler.service.ComponentService) Keytab(org.apache.kerby.kerberos.kerb.keytab.Keytab) MD5Util(com.dtstack.taier.pluginapi.util.MD5Util) QueueService(com.dtstack.taier.scheduler.service.QueueService) DictType(com.dtstack.taier.common.enums.DictType) Xml2JsonUtil(com.dtstack.taier.common.util.Xml2JsonUtil) ComponentVersionUtil(com.dtstack.taier.common.util.ComponentVersionUtil) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult) PublicUtil(com.dtstack.taier.pluginapi.util.PublicUtil) ComponentVO(com.dtstack.taier.scheduler.vo.ComponentVO) DownloadType(com.dtstack.taier.common.enums.DownloadType) WorkerOperator(com.dtstack.taier.scheduler.WorkerOperator) Krb5FileUtil(com.dtstack.taier.scheduler.utils.Krb5FileUtil) Resource(com.dtstack.taier.dao.dto.Resource) ConfigConstant(com.dtstack.taier.pluginapi.constrant.ConfigConstant) JSONObject(com.alibaba.fastjson.JSONObject) PartCluster(com.dtstack.taier.develop.model.PartCluster) ComponentMultiTestResult(com.dtstack.taier.scheduler.impl.pojo.ComponentMultiTestResult) BeanUtils(org.springframework.beans.BeanUtils) java.util(java.util) ZipUtil(com.dtstack.taier.common.util.ZipUtil) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) Part(com.dtstack.taier.develop.model.Part) SftpFileManage(com.dtstack.taier.pluginapi.sftp.SftpFileManage) ClusterFactory(com.dtstack.taier.develop.model.ClusterFactory) EnvironmentContext(com.dtstack.taier.common.env.EnvironmentContext) RdosDefineException(com.dtstack.taier.common.exception.RdosDefineException) BooleanUtils(org.apache.commons.lang3.BooleanUtils) CompletableFuture(java.util.concurrent.CompletableFuture) ClusterMapper(com.dtstack.taier.dao.mapper.ClusterMapper) CustomThreadFactory(com.dtstack.taier.pluginapi.CustomThreadFactory) JSONArray(com.alibaba.fastjson.JSONArray) Lists(com.google.common.collect.Lists) CollectionUtils(org.apache.commons.collections.CollectionUtils) ClientTemplate(com.dtstack.taier.scheduler.impl.pojo.ClientTemplate) ErrorCode(com.dtstack.taier.common.exception.ErrorCode) ComponentConfigUtils(com.dtstack.taier.scheduler.utils.ComponentConfigUtils) ConsoleKerberosMapper(com.dtstack.taier.dao.mapper.ConsoleKerberosMapper) com.dtstack.taier.dao.domain(com.dtstack.taier.dao.domain) Logger(org.slf4j.Logger) MapUtils(org.apache.commons.collections.MapUtils) ExceptionUtil(com.dtstack.taier.pluginapi.exception.ExceptionUtil) XmlFileUtil(com.dtstack.taier.scheduler.utils.XmlFileUtil) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) ScheduleDictService(com.dtstack.taier.scheduler.service.ScheduleDictService) File(java.io.File) ComponentMapper(com.dtstack.taier.dao.mapper.ComponentMapper) SftpConfig(com.dtstack.taier.pluginapi.sftp.SftpConfig) EComponentType(com.dtstack.taier.common.enums.EComponentType) TimeUnit(java.util.concurrent.TimeUnit) ZIP_SUFFIX(com.dtstack.taier.common.constant.CommonConstant.ZIP_SUFFIX) ComponentConfigService(com.dtstack.taier.scheduler.service.ComponentConfigService) PrincipalName(org.apache.kerby.kerberos.kerb.type.base.PrincipalName) Transactional(org.springframework.transaction.annotation.Transactional) RdosDefineException(com.dtstack.taier.common.exception.RdosDefineException) PartCluster(com.dtstack.taier.develop.model.PartCluster) ComponentMultiTestResult(com.dtstack.taier.scheduler.impl.pojo.ComponentMultiTestResult) RdosDefineException(com.dtstack.taier.common.exception.RdosDefineException) IOException(java.io.IOException) CompletableFuture(java.util.concurrent.CompletableFuture) ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult)

Example 3 with ComponentTestResult

use of com.dtstack.taier.pluginapi.pojo.ComponentTestResult in project Taier by DTStack.

the class DtHdfsClient method checkHdfsConnect.

private ComponentTestResult checkHdfsConnect(Config testConnectConf) {
    // 测试hdfs联通性
    ComponentTestResult componentTestResult = new ComponentTestResult();
    try {
        if (null == testConnectConf) {
            componentTestResult.setResult(false);
            componentTestResult.setErrorMsg("配置信息不能你为空");
            return componentTestResult;
        }
        KerberosUtils.login(testConnectConf, () -> {
            FileSystem fs = null;
            try {
                Configuration configuration = this.initYarnConf(testConnectConf.getHadoopConf());
                fs = FileSystem.get(configuration);
                Path path = new Path(configuration.get("yarn.nodemanager.remote-app-log-dir"));
                fs.exists(path);
            } catch (Exception e) {
                componentTestResult.setResult(false);
                componentTestResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
                return componentTestResult;
            } finally {
                if (null != fs) {
                    try {
                        fs.close();
                    } catch (IOException e) {
                        LOG.error("close file system error ", e);
                    }
                }
            }
            componentTestResult.setResult(true);
            return componentTestResult;
        }, KerberosUtils.convertMapConfToConfiguration(testConnectConf.getHadoopConf()));
    } catch (Exception e) {
        LOG.error("close hdfs connect  error ", e);
        componentTestResult.setResult(false);
        componentTestResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
    }
    return componentTestResult;
}
Also used : Path(org.apache.hadoop.fs.Path) Configuration(org.apache.hadoop.conf.Configuration) ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult) FileSystem(org.apache.hadoop.fs.FileSystem) IOException(java.io.IOException) IOException(java.io.IOException) PluginDefineException(com.dtstack.taier.pluginapi.exception.PluginDefineException)

Example 4 with ComponentTestResult

use of com.dtstack.taier.pluginapi.pojo.ComponentTestResult in project Taier by DTStack.

the class HadoopClient method testYarnConnect.

private ComponentTestResult testYarnConnect(ComponentTestResult testResult, Config allConfig) {
    HadoopConf hadoopConf = new HadoopConf();
    hadoopConf.initYarnConf(allConfig.getYarnConf());
    List<NodeReport> nodes = new ArrayList<>();
    try (YarnClient testYarnClient = YarnClient.createYarnClient()) {
        testYarnClient.init(hadoopConf.getYarnConfiguration());
        testYarnClient.start();
        nodes = testYarnClient.getNodeReports(NodeState.RUNNING);
        int totalMemory = 0;
        int totalCores = 0;
        for (NodeReport rep : nodes) {
            totalMemory += rep.getCapability().getMemory();
            totalCores += rep.getCapability().getVirtualCores();
        }
        List<ComponentTestResult.QueueDescription> descriptions = getQueueDescription(null, testYarnClient.getRootQueueInfos());
        testResult.setClusterResourceDescription(new ComponentTestResult.ClusterResourceDescription(nodes.size(), totalMemory, totalCores, descriptions));
    } catch (Exception e) {
        LOG.error("test yarn connect error", e);
        testResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
        return testResult;
    }
    testResult.setResult(true);
    return testResult;
}
Also used : ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult) HadoopConf(com.dtstack.taier.hadoop.util.HadoopConf) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) PluginDefineException(com.dtstack.taier.pluginapi.exception.PluginDefineException)

Example 5 with ComponentTestResult

use of com.dtstack.taier.pluginapi.pojo.ComponentTestResult in project Taier by DTStack.

the class DtYarnClient method testYarnConnect.

private ComponentTestResult testYarnConnect(ComponentTestResult testResult, Config allConfig) {
    try {
        HadoopConf hadoopConf = new HadoopConf();
        hadoopConf.initYarnConf(allConfig.getYarnConf());
        YarnClient testYarnClient = YarnClient.createYarnClient();
        testYarnClient.init(hadoopConf.getYarnConfiguration());
        testYarnClient.start();
        List<NodeReport> nodes = testYarnClient.getNodeReports(NodeState.RUNNING);
        int totalMemory = 0;
        int totalCores = 0;
        for (NodeReport rep : nodes) {
            totalMemory += rep.getCapability().getMemory();
            totalCores += rep.getCapability().getVirtualCores();
        }
        boolean isFullPath = hadoopConf.getYarnConfiguration().getBoolean(ConfigConstrant.IS_FULL_PATH_KEY, false);
        String rootQueueName = isFullPath ? getRootQueueName(testYarnClient) : "";
        List<ComponentTestResult.QueueDescription> descriptions = getQueueDescription(rootQueueName, testYarnClient.getRootQueueInfos(), isFullPath);
        testResult.setClusterResourceDescription(new ComponentTestResult.ClusterResourceDescription(nodes.size(), totalMemory, totalCores, descriptions));
    } catch (Exception e) {
        LOG.error("test yarn connect error", e);
        throw new PluginDefineException(e);
    }
    testResult.setResult(true);
    return testResult;
}
Also used : YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) IOException(java.io.IOException) PluginDefineException(com.dtstack.taier.pluginapi.exception.PluginDefineException) ComponentTestResult(com.dtstack.taier.pluginapi.pojo.ComponentTestResult) PluginDefineException(com.dtstack.taier.pluginapi.exception.PluginDefineException) HadoopConf(com.dtstack.taier.yarn.util.HadoopConf) NodeReport(org.apache.hadoop.yarn.api.records.NodeReport)

Aggregations

ComponentTestResult (com.dtstack.taier.pluginapi.pojo.ComponentTestResult)15 PluginDefineException (com.dtstack.taier.pluginapi.exception.PluginDefineException)8 IOException (java.io.IOException)8 RdosDefineException (com.dtstack.taier.common.exception.RdosDefineException)3 JSONObject (com.alibaba.fastjson.JSONObject)2 PartCluster (com.dtstack.taier.develop.model.PartCluster)2 HadoopConf (com.dtstack.taier.hadoop.util.HadoopConf)2 SftpConfig (com.dtstack.taier.pluginapi.sftp.SftpConfig)2 SftpFileManage (com.dtstack.taier.pluginapi.sftp.SftpFileManage)2 Configuration (org.apache.hadoop.conf.Configuration)2 YarnClient (org.apache.hadoop.yarn.client.api.YarnClient)2 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)2 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)2 JSONArray (com.alibaba.fastjson.JSONArray)1 ZIP_SUFFIX (com.dtstack.taier.common.constant.CommonConstant.ZIP_SUFFIX)1 DictType (com.dtstack.taier.common.enums.DictType)1 DownloadType (com.dtstack.taier.common.enums.DownloadType)1 EComponentType (com.dtstack.taier.common.enums.EComponentType)1 EnvironmentContext (com.dtstack.taier.common.env.EnvironmentContext)1 ErrorCode (com.dtstack.taier.common.exception.ErrorCode)1