Examples with PluginDefineException com.dtstack.taier.pluginapi.exception.PluginDefineException used on opensource projects

Example 31 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class AcceptedApplicationMonitor method run.

@Override
public void run() {
    try (YarnClient yarnClient = KerberosUtils.login(config, () -> {
        YarnClient client = YarnClient.createYarnClient();
        client.init(yarnConf);
        client.start();
        return client;
    }, yarnConf)) {
        EnumSet<YarnApplicationState> enumSet = EnumSet.noneOf(YarnApplicationState.class);
        enumSet.add(YarnApplicationState.ACCEPTED);
        if (yarnClient == null) {
            throw new PluginDefineException("AcceptedApplicationMonitor init yarnClient fail");
        }
        List<ApplicationReport> acceptedApps = yarnClient.getApplications(enumSet).stream().filter(report -> report.getQueue().endsWith(queueName)).collect(Collectors.toList());
        for (ApplicationReport report : acceptedApps) {
            long startTime = report.getStartTime();
            long currentTime = System.currentTimeMillis();
            if (currentTime - startTime > THRESHOLD) {
                ApplicationId appId = report.getApplicationId();
                yarnClient.killApplication(appId);
            }
        }
    } catch (Exception e) {
        logger.error("Monitor Accepted Application Exception: " + e.getMessage());
    }
}

Example 32 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SftpFileManage method downloadDir.

/**
 * download 后自动释放连接
 *
 * @param remoteDir
 * @param localDir
 * @return
 */
public boolean downloadDir(String remoteDir, String localDir) {
    ChannelSftp channelSftp = null;
    try {
        // 检查并创建本地文件目录
        File localDirPath = new File(localDir);
        if (!localDirPath.exists()) {
            boolean mkdirs = localDirPath.mkdirs();
            LOGGER.info("local file localDir {}  mkdir {} :", localDir, mkdirs);
        }
        channelSftp = getChannelSftp();
        Vector files = channelSftp.ls(remoteDir);
        if (files == null) {
            return false;
        }
        // 递归下载文件
        for (Iterator<ChannelSftp.LsEntry> iterator = files.iterator(); iterator.hasNext(); ) {
            ChannelSftp.LsEntry str = iterator.next();
            String filename = str.getFilename();
            if (".".equals(filename) || "..".equals(filename)) {
                continue;
            }
            SftpATTRS attrs = str.getAttrs();
            boolean isdir = attrs.isDir();
            String localFilePath = localDir + "/" + filename;
            String ftpFilePath;
            if (channelSftp.stat(remoteDir).isDir()) {
                ftpFilePath = remoteDir + "/" + filename;
            } else {
                ftpFilePath = remoteDir;
            }
            if (isdir) {
                File dir2 = new File(localFilePath);
                if (!dir2.exists()) {
                    LOGGER.info("local file path mkdir :", localFilePath);
                    dir2.mkdir();
                }
                downloadDir(ftpFilePath, localFilePath);
            } else {
                downloadFile(ftpFilePath, localFilePath, channelSftp);
            }
        }
        return true;
    } catch (Exception e) {
        LOGGER.error("sftp downloadDir error {}", e);
        throw new PluginDefineException(e);
    } finally {
        close(channelSftp);
    }
}

Example 33 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class SftpFileManage method listFile.

public Vector listFile(String remotePath) {
    ChannelSftp channelSftp = null;
    try {
        channelSftp = getChannelSftp();
        Vector vector = channelSftp.ls(remotePath);
        return vector;
    } catch (SftpException e) {
        LOGGER.error("listFile  error", e);
        throw new PluginDefineException(e);
    } finally {
        close(channelSftp);
    }
}

Example 34 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class UrlUtil method getHttpRootUrl.

/**
 * 返回URL请求的 ${域名:port}部分
 * @return
 */
public static String getHttpRootUrl(String url) {
    Matcher matcher = URLPattern.matcher(url);
    if (!matcher.find()) {
        throw new PluginDefineException(String.format("url:%s is not regular HTTP_URL", url));
    }
    String protocol = matcher.group(1) == null ? "http" : matcher.group(1);
    String hostName = matcher.group(2);
    String port = matcher.group(3);
    if (port == null) {
        return protocol + "://" + hostName;
    } else {
        return protocol + "://" + hostName + ":" + port;
    }
}

Example 35 with PluginDefineException

use of com.dtstack.taier.pluginapi.exception.PluginDefineException in project Taier by DTStack.

the class HadoopClient method downloadHdfsFile.

private void downloadHdfsFile(String from, String to) throws IOException {
    try {
        KerberosUtils.login(config, () -> {
            try {
                File toFile = new File(to);
                if (!toFile.getParentFile().exists()) {
                    Files.createParentDirs(toFile);
                }
                Path hdfsFilePath = new Path(from);
                // 读取文件
                InputStream is = FileSystem.get(conf).open(hdfsFilePath);
                // 保存到本地
                IOUtils.copyBytes(is, new FileOutputStream(toFile), 2048, true);
            } catch (Exception e) {
                throw new PluginDefineException(e);
            }
            return null;
        }, conf);
    } catch (Exception e) {
        LOG.error("", e);
        throw new PluginDefineException(e);
    }
}