Examples with SftpFileManage com.dtstack.taier.pluginapi.sftp.SftpFileManage used on opensource projects

Example 6 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class KerberosUtils method getKeytabPath.

public static String getKeytabPath(BaseConfig config) {
    String fileName = config.getPrincipalFile();
    String remoteDir = config.getRemoteDir();
    String localDir = USER_DIR + remoteDir;
    File path = new File(localDir);
    if (!path.exists()) {
        path.mkdirs();
    }
    SftpFileManage sftpFileManage = SftpFileManage.getSftpManager(config.getSftpConf());
    logger.info("fileName:{}, remoteDir:{}, localDir:{}, sftpConf:{}", fileName, remoteDir, localDir, config.getSftpConf());
    String keytabPath = sftpFileManage.cacheOverloadFile(fileName, remoteDir, localDir);
    logger.info("keytabPath:{}", keytabPath);
    return keytabPath;
}

Example 7 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class DummyClient method testConnect.

@Override
public ComponentTestResult testConnect(String pluginInfo) {
    ComponentTestResult componentTestResult = new ComponentTestResult();
    try {
        SftpConfig sftpConfig = PublicUtil.jsonStrToObject(pluginInfo, SftpConfig.class);
        // check sftpConfig 准确性
        SftpFileManage sftpFileManage = SftpFileManage.getSftpManager(sftpConfig);
        // 测试路径是否存在
        Vector res = sftpFileManage.listFile(sftpConfig.getPath());
        if (null != res) {
            componentTestResult.setResult(true);
        }
    } catch (Exception e) {
        componentTestResult.setErrorMsg(ExceptionUtil.getErrorMessage(e));
        componentTestResult.setResult(false);
    }
    return componentTestResult;
}

Example 8 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class KerberosUtils method getKerberosFile.

public static synchronized String[] getKerberosFile(BaseConfig config, String localDir) {
    String keytabFileName = config.getPrincipalFile();
    String krb5FileName = config.getKrbName();
    String remoteDir = config.getRemoteDir();
    Boolean isMergeKrb5 = StringUtils.isNotEmpty(config.getMergeKrbContent());
    if (StringUtils.isEmpty(localDir)) {
        localDir = ConfigConstant.LOCAL_KEYTAB_DIR_PARENT + remoteDir;
    }
    File localDirPath = new File(localDir);
    if (!localDirPath.exists()) {
        localDirPath.mkdirs();
    }
    String keytabPath = "";
    String krb5ConfPath = "";
    boolean isOverrideDownLoad = checkLocalCache(config.getKerberosFileTimestamp(), localDirPath);
    if (isOverrideDownLoad) {
        SftpFileManage sftpFileManage = SftpFileManage.getSftpManager(config.getSftpConf());
        keytabPath = sftpFileManage.cacheOverloadFile(keytabFileName, remoteDir, localDir);
        if (isMergeKrb5) {
            krb5ConfPath = localDir + ConfigConstant.SP + ConfigConstant.MERGE_KRB5_NAME;
            try {
                Files.write(Paths.get(krb5ConfPath), Collections.singleton(config.getMergeKrbContent()));
            } catch (IOException e) {
                throw new PluginDefineException(e);
            }
        } else {
            krb5ConfPath = sftpFileManage.cacheOverloadFile(krb5FileName, remoteDir, localDir);
        }
        writeTimeLockFile(config.getKerberosFileTimestamp(), localDir);
    } else {
        keytabPath = localDir + File.separator + keytabFileName;
        krb5ConfPath = localDir + File.separator + krb5FileName;
    }
    logger.info("Get keytabPath: {}, krb5ConfPath: {}", keytabPath, krb5ConfPath);
    return new String[] { keytabPath, krb5ConfPath };
}

Example 9 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class KerberosUtils method login.

/**
 * 重载login方法 ，增加IsCreateNewUGI 来检查是否重新create ugi
 * @param config
 * @param supplier
 * @param configuration
 * @param isCreateNewUGI
 * @param <T>
 * @return
 * @throws Exception
 */
public static <T> T login(BaseConfig config, Supplier<T> supplier, Configuration configuration, boolean isCreateNewUGI) throws Exception {
    if (Objects.isNull(config) || !config.isOpenKerberos()) {
        return supplier.get();
    }
    String fileName = config.getPrincipalFile();
    String remoteDir = config.getRemoteDir();
    String localDir = ConfigConstant.LOCAL_KEYTAB_DIR_PARENT + remoteDir;
    String finalKrb5ConfPath;
    String finalPrincipal;
    String finalKeytabPath;
    String threadName;
    Boolean isMergeKrb5;
    File localDirPath = new File(localDir);
    if (!localDirPath.exists()) {
        localDirPath.mkdirs();
    }
    logger.info("fileName:{}, remoteDir:{}, localDir:{}, sftpConf:{}", fileName, remoteDir, localDir, config.getSftpConf());
    try {
        UserGroupInformation ugi;
        String segmentName = segment.computeIfAbsent(remoteDir, key -> {
            return new String(remoteDir);
        });
        synchronized (segmentName) {
            String keytabPath = "";
            String krb5ConfPath = "";
            String krb5ConfName = config.getKrbName();
            isMergeKrb5 = StringUtils.isNotEmpty(config.getMergeKrbContent());
            // 本地文件是否和服务器时间一致 一致使用本地缓存
            boolean isOverrideDownLoad = checkLocalCache(config.getKerberosFileTimestamp(), localDirPath);
            if (isOverrideDownLoad) {
                SftpFileManage sftpFileManage = SftpFileManage.getSftpManager(config.getSftpConf());
                keytabPath = sftpFileManage.cacheOverloadFile(fileName, remoteDir, localDir);
                krb5ConfPath = sftpFileManage.cacheOverloadFile(krb5ConfName, config.getRemoteDir(), localDir);
                if (isMergeKrb5) {
                    krb5ConfPath = localDir + ConfigConstant.SP + ConfigConstant.MERGE_KRB5_NAME;
                    Files.write(Paths.get(krb5ConfPath), Collections.singleton(config.getMergeKrbContent()));
                }
                writeTimeLockFile(config.getKerberosFileTimestamp(), localDir);
            } else {
                keytabPath = localDir + File.separator + fileName;
                if (isMergeKrb5) {
                    krb5ConfPath = localDir + ConfigConstant.SP + ConfigConstant.MERGE_KRB5_NAME;
                } else {
                    krb5ConfPath = localDir + ConfigConstant.SP + krb5ConfName;
                }
            }
            finalKrb5ConfPath = krb5ConfPath;
            finalKeytabPath = keytabPath;
            threadName = Thread.currentThread().getName();
            String principal = config.getPrincipal();
            if (StringUtils.isEmpty(principal)) {
                principal = segment.computeIfAbsent(threadName, k -> {
                    return KerberosUtils.getPrincipal(finalKeytabPath);
                });
            }
            finalPrincipal = principal;
            logger.info("kerberos login, principal:{}, keytabPath:{}, krb5ConfPath:{}", principal, keytabPath, krb5ConfPath);
            /*
                 * 如果用已经带有token的ugi进行认证时，在HDFS DELEGATION TOKEN那里会出现认证错误
                 * 如果是SPARK 在这里先每次创建UGI进行避开
                 */
            if (isCreateNewUGI) {
                ugi = retryCreateUGIIfMerge(finalKrb5ConfPath, configuration, finalPrincipal, finalKeytabPath, config.getKrbName(), isMergeKrb5);
            } else {
                ugi = ugiMap.computeIfAbsent(threadName, k -> retryCreateUGIIfMerge(finalKrb5ConfPath, configuration, finalPrincipal, finalKeytabPath, config.getKrbName(), isMergeKrb5));
            }
            KerberosTicket ticket = getTGT(ugi);
            if (!checkTGT(ticket) || isOverrideDownLoad) {
                logger.info("Relogin after the ticket expired, principal: {}, current thread: {}", principal, Thread.currentThread().getName());
                ugi = retryCreateUGIIfMerge(finalKrb5ConfPath, configuration, finalPrincipal, finalKeytabPath, config.getKrbName(), isMergeKrb5);
                if (!isCreateNewUGI) {
                    ugiMap.put(threadName, ugi);
                }
            }
            logger.info("userGroupInformation current user = {} ugi user  = {} ", UserGroupInformation.getCurrentUser(), ugi.getUserName());
        }
        Preconditions.checkNotNull(ugi, "UserGroupInformation is null");
        return KerberosUtils.retryLoginKerberosWithCallBack(ugi, supplier, finalKrb5ConfPath, configuration, finalPrincipal, finalKeytabPath, threadName, config.getKrbName(), isMergeKrb5);
    } catch (Exception e) {
        throw new PluginDefineException(e.getMessage());
    }
}