Examples with PluginDefineException com.dtstack.taier.pluginapi.exception.PluginDefineException used on opensource projects

Example 36 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class DtYarnClient method getClusterResource.

@Override
public ClusterResource getClusterResource() {
    ClusterResource clusterResource = new ClusterResource();
    try {
        KerberosUtils.login(config, () -> {
            YarnClient resourceClient = null;
            try {
                resourceClient = YarnClient.createYarnClient();
                resourceClient.init(configuration);
                resourceClient.start();
                List<NodeReport> nodes = resourceClient.getNodeReports(NodeState.RUNNING);
                List<ClusterResource.NodeDescription> clusterNodes = new ArrayList<>();
                Integer totalMem = 0;
                Integer totalCores = 0;
                Integer usedMem = 0;
                Integer usedCores = 0;
                for (NodeReport rep : nodes) {
                    ClusterResource.NodeDescription node = new ClusterResource.NodeDescription();
                    String nodeName = rep.getHttpAddress().split(":")[0];
                    node.setNodeName(nodeName);
                    node.setMemory(rep.getCapability().getMemory());
                    node.setUsedMemory(rep.getUsed().getMemory());
                    node.setUsedVirtualCores(rep.getUsed().getVirtualCores());
                    node.setVirtualCores(rep.getCapability().getVirtualCores());
                    clusterNodes.add(node);
                    // 计算集群资源总量和使用量
                    Resource capability = rep.getCapability();
                    Resource used = rep.getUsed();
                    totalMem += capability.getMemory();
                    totalCores += capability.getVirtualCores();
                    usedMem += used.getMemory();
                    usedCores += used.getVirtualCores();
                }
                ClusterResource.ResourceMetrics metrics = createResourceMetrics(totalMem, usedMem, totalCores, usedCores);
                clusterResource.setNodes(clusterNodes);
                String webAddress = getYarnWebAddress(resourceClient);
                String schedulerUrl = String.format(YARN_SCHEDULER_FORMAT, webAddress);
                String schedulerInfoMsg = PoolHttpClient.get(schedulerUrl, null);
                JSONObject schedulerInfo = JSONObject.parseObject(schedulerInfoMsg);
                if (schedulerInfo.containsKey("scheduler")) {
                    clusterResource.setScheduleInfo(schedulerInfo.getJSONObject("scheduler").getJSONObject("schedulerInfo"));
                }
                clusterResource.setQueues(getQueueResource(resourceClient));
                clusterResource.setResourceMetrics(metrics);
            } catch (Exception e) {
                LOG.error("close reource error ", e);
            } finally {
                if (null != resourceClient) {
                    try {
                        resourceClient.close();
                    } catch (IOException e) {
                        LOG.error("close reource error ", e);
                    }
                }
            }
            return clusterResource;
        }, configuration);
    } catch (Exception e) {
        throw new PluginDefineException(e.getMessage());
    }
    return clusterResource;
}

Example 37 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class AbstractConnFactory method init.

public void init(Properties properties) throws ClassNotFoundException {
    synchronized (AbstractConnFactory.class) {
        if (isFirstLoaded.get()) {
            Class.forName(driverName);
            isFirstLoaded.set(false);
        }
    }
    jdbcUrl = MathUtil.getString(properties.get(ConfigConstant.JDBCURL));
    username = MathUtil.getString(properties.get(ConfigConstant.USERNAME));
    password = MathUtil.getString(properties.get(ConfigConstant.PASSWORD));
    Preconditions.checkNotNull(jdbcUrl, "db url can't be null");
    try {
        String propStr = PublicUtil.objToString(properties);
        baseConfig = PublicUtil.jsonStrToObject(propStr, BaseConfig.class);
        // 非kerberos 不进行yarnConf初始化
        if (baseConfig.isOpenKerberos() && null != properties.get("yarnConf")) {
            Map<String, Object> yarnMap = (Map<String, Object>) properties.get("yarnConf");
            yarnConf = KerberosUtils.convertMapConfToConfiguration(yarnMap);
        }
        testConn();
    } catch (Exception e) {
        throw new PluginDefineException("get conn exception:" + e.toString());
    }
}

Example 38 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class AbstractRdbsClient method executeQuery.

@Override
public List<List<Object>> executeQuery(String sql, String database) {
    Statement statement = null;
    ResultSet res = null;
    Connection conn = null;
    List<List<Object>> result = Lists.newArrayList();
    try {
        if (StringUtils.isBlank(sql)) {
            return null;
        }
        conn = connFactory.getConn();
        statement = conn.createStatement();
        if (StringUtils.isNotBlank(database)) {
            statement.execute("use " + database);
        }
        if (statement.execute(sql)) {
            res = statement.getResultSet();
            int columns = res.getMetaData().getColumnCount();
            List<Object> cloumnName = Lists.newArrayList();
            int timeStamp = 0;
            SimpleDateFormat dateFormat = null;
            for (int i = 1; i <= columns; ++i) {
                String name = res.getMetaData().getColumnName(i);
                if (name.contains(".")) {
                    name = name.split("\\.")[1];
                }
                cloumnName.add(name);
            }
            result.add(cloumnName);
            while (res.next()) {
                List<Object> objects = Lists.newArrayList();
                for (int i = 1; i <= columns; ++i) {
                    if (i == timeStamp && null != dateFormat) {
                        objects.add(dateFormat.format(res.getObject(i)));
                    } else {
                        objects.add(res.getObject(i));
                    }
                }
                result.add(objects);
            }
        }
    } catch (Exception e) {
        LOG.error("execue sql {} error", sql, e);
        throw new PluginDefineException(e);
    } finally {
        try {
            if (res != null) {
                res.close();
            }
            if (statement != null) {
                statement.close();
            }
            if (null != conn) {
                conn.close();
                ;
            }
        } catch (Throwable var18) {
            LOG.error("", var18);
        }
    }
    return result;
}

Example 39 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class DtHdfsClient method uploadStringToHdfs.

/**
 * 上传文件到hdfs中
 * @param bytes
 * @param hdfsPath 文件路径
 * @return
 */
@Override
public String uploadStringToHdfs(String bytes, String hdfsPath) {
    try {
        configuration = this.initYarnConf(config.getYarnConf());
        return KerberosUtils.login(config, () -> {
            FileSystem fs = null;
            try {
                ByteArrayInputStream is = new ByteArrayInputStream(bytes.getBytes());
                fs = FileSystem.get(configuration);
                Path destP = new Path(hdfsPath);
                FSDataOutputStream os = fs.create(destP);
                IOUtils.copyBytes(is, os, 4096, true);
            } catch (IOException e) {
                LOG.error("submit file {} to hdfs error", hdfsPath, e);
                throw new PluginDefineException("上传文件失败", e);
            } finally {
                if (Objects.nonNull(fs)) {
                    try {
                        fs.close();
                    } catch (IOException e) {
                    }
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("submit file {} to hdfs success.", hdfsPath);
            }
            return configuration.get("fs.defaultFS") + hdfsPath;
        }, configuration);
    } catch (Exception e) {
        throw new PluginDefineException("上传文件失败", e);
    }
}

Example 40 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SparkYarnClient method addEnv2SparkConf.

/**
 * 1.parse cmd.
 * 2.add key-value into container envs.
 * @param cmdStr
 */
private void addEnv2SparkConf(String cmdStr, SparkConf sparkConf) {
    if (cmdStr == null) {
        return;
    }
    try {
        // envJson is encode, we need decode it.
        cmdStr = URLDecoder.decode(cmdStr, "UTF-8");
        logger.info("cmdStr decoded is : " + cmdStr);
        Map<String, Object> envMap = JSON.parseObject(cmdStr.trim());
        Iterator entries = envMap.entrySet().iterator();
        while (entries.hasNext()) {
            Map.Entry entry = (Map.Entry) entries.next();
            String key = (String) entry.getKey();
            String value;
            if (AppEnvConstant.MODEL_PARAM.equals(key)) {
                value = URLEncoder.encode((String) entry.getValue(), "UTF-8");
            } else {
                value = (String) entry.getValue();
            }
            // add prefix for app env, make it easier to recognize.
            sparkConf.setExecutorEnv(key, value);
        }
    } catch (Exception e) {
        String message = String.format("Could't parse {%s} to json format. Reason : {%s}", cmdStr, e.getMessage());
        logger.error(message);
        throw new PluginDefineException(message, e);
    }
}