Examples with HadoopConf com.dtstack.taier.hadoop.util.HadoopConf used on opensource projects

Example 1 with HadoopConf

use of com.dtstack.taier.hadoop.util.HadoopConf in project Taier by DTStack.

the class HadoopClient method testYarnConnect.

private ComponentTestResult testYarnConnect(ComponentTestResult testResult, Config allConfig) {
    HadoopConf hadoopConf = new HadoopConf();
    hadoopConf.initYarnConf(allConfig.getYarnConf());
    List<NodeReport> nodes = new ArrayList<>();
    try (YarnClient testYarnClient = YarnClient.createYarnClient()) {
        testYarnClient.init(hadoopConf.getYarnConfiguration());
        testYarnClient.start();
        nodes = testYarnClient.getNodeReports(NodeState.RUNNING);
        int totalMemory = 0;
        int totalCores = 0;
        for (NodeReport rep : nodes) {
            totalMemory += rep.getCapability().getMemory();
            totalCores += rep.getCapability().getVirtualCores();
        }
        List<ComponentTestResult.QueueDescription> descriptions = getQueueDescription(null, testYarnClient.getRootQueueInfos());
        testResult.setClusterResourceDescription(new ComponentTestResult.ClusterResourceDescription(nodes.size(), totalMemory, totalCores, descriptions));
    } catch (Exception e) {
        LOG.error("test yarn connect error", e);
        testResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
        return testResult;
    }
    testResult.setResult(true);
    return testResult;
}

Example 2 with HadoopConf

use of com.dtstack.taier.hadoop.util.HadoopConf in project Taier by DTStack.

the class HadoopClient method checkHdfsConnect.

private ComponentTestResult checkHdfsConnect(Config testConnectConf) {
    // 测试hdfs联通性
    ComponentTestResult componentTestResult = new ComponentTestResult();
    try {
        if (null == testConnectConf) {
            componentTestResult.setResult(false);
            componentTestResult.setErrorMsg("配置信息不能你为空");
            return componentTestResult;
        }
        KerberosUtils.login(testConnectConf, () -> {
            HadoopConf hadoopConf = new HadoopConf();
            hadoopConf.initHadoopConf(testConnectConf.getHadoopConf());
            Configuration configuration = hadoopConf.getConfiguration();
            FileSystem fs = null;
            try {
                fs = FileSystem.get(configuration);
            } catch (Exception e) {
                componentTestResult.setResult(false);
                componentTestResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
                return componentTestResult;
            } finally {
                if (null != fs) {
                    try {
                        fs.close();
                    } catch (IOException e) {
                        LOG.error("close file system error ", e);
                    }
                }
            }
            componentTestResult.setResult(true);
            return componentTestResult;
        }, KerberosUtils.convertMapConfToConfiguration(testConnectConf.getHadoopConf()));
    } catch (Exception e) {
        LOG.error("close hdfs connect  error ", e);
        componentTestResult.setResult(false);
        componentTestResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
    }
    return componentTestResult;
}

Example 3 with HadoopConf

use of com.dtstack.taier.hadoop.util.HadoopConf in project Taier by DTStack.

the class HadoopClient method init.

@Override
public void init(Properties prop) throws Exception {
    LOG.info("hadoop client init...");
    String configStr = PublicUtil.objToString(prop);
    config = PublicUtil.jsonStrToObject(configStr, Config.class);
    HadoopConf customerConf = new HadoopConf();
    customerConf.initHadoopConf(config.getHadoopConf());
    customerConf.initYarnConf(config.getYarnConf());
    conf = customerConf.getYarnConfiguration();
    HadoopConfTool.setFsHdfsImplDisableCache(conf);
    conf.set("mapreduce.framework.name", "yarn");
    conf.set("mapreduce.map.memory.mb", "1024");
    conf.set("mapreduce.reduce.memory.mb", "1024");
    conf.setBoolean("mapreduce.app-submission.cross-platform", true);
    setHadoopUserName(config);
    yarnClient = buildYarnClient();
    LOG.info("UGI info: " + UserGroupInformation.getCurrentUser());
}