Examples with SftpFileManage com.dtstack.taier.pluginapi.sftp.SftpFileManage used on opensource projects

Example 1 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class ConsoleComponentService method mergeKrb5.

private synchronized String mergeKrb5(List<KerberosConfig> kerberosConfigs) {
    String mergeKrb5Content = "";
    if (CollectionUtils.isEmpty(kerberosConfigs)) {
        LOGGER.error("KerberosConfigs is null");
        return mergeKrb5Content;
    }
    String mergeDirPath = ConfigConstant.LOCAL_KRB5_MERGE_DIR_PARENT + ConfigConstant.SP + UUID.randomUUID();
    List<Long> clusterDownloadRecords = new ArrayList<>();
    try {
        String oldMergeKrb5Content = null;
        String mergeKrb5Path = mergeDirPath + ConfigConstant.SP + ConfigConstant.MERGE_KRB5_NAME;
        for (KerberosConfig kerberosConfig : kerberosConfigs) {
            String krb5Name = kerberosConfig.getKrbName();
            String remotePath = kerberosConfig.getRemotePath();
            Long clusterId = kerberosConfig.getClusterId();
            Integer componentCode = kerberosConfig.getComponentType();
            if (StringUtils.isNotEmpty(kerberosConfig.getMergeKrbContent()) && StringUtils.isEmpty(oldMergeKrb5Content)) {
                oldMergeKrb5Content = kerberosConfig.getMergeKrbContent();
            }
            String remoteKrb5Path = remotePath + ConfigConstant.SP + krb5Name;
            String localKrb5Path = mergeDirPath + remoteKrb5Path;
            try {
                String sftpComponent = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, String.class, null);
                SftpConfig sftpConfig = getSFTPConfig(sftpComponent, componentCode, "");
                SftpFileManage sftpFileManage = sftpFileManageBean.retrieveSftpManager(sftpConfig);
                if (clusterDownloadRecords.contains(clusterId)) {
                    continue;
                }
                boolean downRes = sftpFileManage.downloadFile(remoteKrb5Path, localKrb5Path);
                LOGGER.info("download remoteKrb5Path[{}] result {}", remoteKrb5Path, downRes);
                if (downRes) {
                    clusterDownloadRecords.add(clusterId);
                    if (!new File(mergeKrb5Path).exists()) {
                        FileUtils.copyFile(new File(localKrb5Path), new File(mergeKrb5Path));
                        mergeKrb5Content = Krb5FileUtil.convertMapToString(Krb5FileUtil.readKrb5ByPath(mergeKrb5Path));
                        continue;
                    }
                    mergeKrb5Content = Krb5FileUtil.mergeKrb5ContentByPath(mergeKrb5Path, localKrb5Path);
                }
            } catch (Exception e) {
                LOGGER.error("merge krb5.conf[{}] error : {}", localKrb5Path, e.getMessage());
            }
        }
        if (StringUtils.isNotEmpty(oldMergeKrb5Content)) {
            mergeKrb5Content = Krb5FileUtil.resetMergeKrb5Content(oldMergeKrb5Content, mergeKrb5Content);
        }
        LOGGER.info("mergeKrb5Content is {}", mergeKrb5Content);
        for (KerberosConfig kerberosConfig : kerberosConfigs) {
            kerberosConfig.setMergeKrbContent(mergeKrb5Content);
            consoleKerberosMapper.updateById(kerberosConfig);
            LOGGER.info("Krb5[{}/krb5.conf] merge successed!", kerberosConfig.getRemotePath());
        }
    } catch (Exception e) {
        LOGGER.error("Merge krb5 error! {}", e.getMessage());
    } finally {
        try {
            File mergeDir = new File(mergeDirPath);
            FileUtils.deleteDirectory(mergeDir);
        } catch (Exception e) {
        }
    }
    return mergeKrb5Content;
}

Example 2 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class ConsoleComponentService method uploadResourceToSftp.

private String uploadResourceToSftp(Long clusterId, List<Resource> resources, String kerberosFileName, SftpConfig sftpConfig, Component addComponent, Component dbComponent, String principals, String principal) {
    // 上传配置文件到sftp 供后续下载
    SftpFileManage sftpFileManage = sftpFileManageBean.retrieveSftpManager(sftpConfig);
    String md5sum = "";
    String remoteDir = sftpConfig.getPath() + File.separator + this.buildSftpPath(clusterId, addComponent.getComponentTypeCode());
    for (Resource resource : resources) {
        if (!resource.getFileName().equalsIgnoreCase(kerberosFileName) || StringUtils.isBlank(kerberosFileName)) {
            addComponent.setUploadFileName(resource.getFileName());
        }
        try {
            if (resource.getFileName().equalsIgnoreCase(kerberosFileName)) {
                // 更新Kerberos信息
                this.updateComponentKerberosFile(clusterId, addComponent, sftpFileManage, remoteDir, resource, principals, principal);
            } else {
                LOGGER.info("start upload hadoop config file:{}", kerberosFileName);
                this.updateComponentConfigFile(dbComponent, sftpFileManage, remoteDir, resource);
                if (EComponentType.HDFS.getTypeCode().equals(addComponent.getComponentTypeCode())) {
                    String xmlZipLocation = resource.getUploadedFileName();
                    md5sum = MD5Util.getFileMd5String(new File(xmlZipLocation));
                    this.updateConfigToSftpPath(clusterId, sftpConfig, sftpFileManage, resource, null, addComponent.getComponentTypeCode());
                }
                if (EComponentType.YARN.getTypeCode().equals(addComponent.getComponentTypeCode())) {
                    List<ComponentConfig> clientTemplates = scheduleDictService.loadExtraComponentConfig(addComponent.getVersionValue(), addComponent.getComponentTypeCode());
                    this.updateConfigToSftpPath(clusterId, sftpConfig, sftpFileManage, resource, clientTemplates, addComponent.getComponentTypeCode());
                }
            }
        } catch (Exception e) {
            LOGGER.error("update component resource {}  error", resource.getUploadedFileName(), e);
            if (e instanceof RdosDefineException) {
                throw (RdosDefineException) e;
            } else {
                throw new RdosDefineException("Failed to update component");
            }
        } finally {
            try {
                FileUtils.forceDelete(new File(resource.getUploadedFileName()));
            } catch (IOException e) {
                LOGGER.error("delete upload file {} error", resource.getUploadedFileName(), e);
            }
        }
    }
    return md5sum;
}

Example 3 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class ConsoleComponentService method updateComponentKerberosFile.

/**
 * 解压kerberos文件到本地 并上传至sftp
 * * @param clusterId
 *
 * @param addComponent
 * @param remoteDir
 * @param resource
 * @return
 */
private String updateComponentKerberosFile(Long clusterId, Component addComponent, SftpFileManage sftpFileManage, String remoteDir, Resource resource, String principals, String principal) {
    File keyTabFile = null;
    File krb5ConfFile = null;
    String remoteDirKerberos = remoteDir + File.separator + KERBEROS;
    if (resource != null) {
        // kerberos认证文件 远程删除 kerberos下的文件
        LOGGER.info("updateComponentKerberosFile remote path:{}", remoteDirKerberos);
        // 删除本地文件夹
        String kerberosPath = this.getLocalKerberosPath(clusterId, addComponent.getComponentTypeCode());
        try {
            FileUtils.deleteDirectory(new File(kerberosPath));
        } catch (IOException e) {
            LOGGER.error("delete old kerberos directory {} error", kerberosPath, e);
        }
        // 解压到本地
        List<File> files = ZipUtil.upzipFile(resource.getUploadedFileName(), kerberosPath);
        if (CollectionUtils.isEmpty(files)) {
            throw new RdosDefineException("Hadoop-Kerberos file decompression error");
        }
        keyTabFile = files.stream().filter(f -> f.getName().endsWith(KEYTAB_SUFFIX)).findFirst().orElse(null);
        krb5ConfFile = files.stream().filter(f -> f.getName().equalsIgnoreCase(KRB5_CONF)).findFirst().orElse(null);
        if (keyTabFile == null) {
            throw new RdosDefineException("There must be a keytab file in the zip file of the uploaded Hadoop-Kerberos file, please add the keytab file");
        }
        LOGGER.info("fileKeyTab Unzip fileName:{}", keyTabFile.getAbsolutePath());
        if (krb5ConfFile == null) {
            throw new RdosDefineException("There must be a krb5.conf file in the zip file of the uploaded Hadoop-Kerberos file, please add the krb5.conf file");
        }
        LOGGER.info("conf Unzip fileName:{}", krb5ConfFile.getAbsolutePath());
        // 获取principal
        List<PrincipalName> principalLists = this.getPrincipal(keyTabFile);
        principal = parsePrincipal(principal, principalLists);
        if (StringUtils.isEmpty(principals)) {
            List<String> principalNames = new ArrayList<>();
            for (PrincipalName principalName : principalLists) {
                principalNames.add(principalName.getName());
            }
            principals = StringUtils.join(principalNames, ",");
        }
        // 删除sftp原来kerberos 的文件夹
        sftpFileManage.deleteDir(remoteDirKerberos);
        // 上传kerberos解压后的文件
        for (File file : files) {
            LOGGER.info("upload sftp file:{}", file.getAbsolutePath());
            sftpFileManage.uploadFile(remoteDirKerberos, file.getPath());
        }
    }
    String versionName = addComponent.getVersionName();
    String componentVersion = "";
    if (StringUtils.isNotBlank(versionName) && ComponentVersionUtil.isMultiVersionComponent(addComponent.getComponentTypeCode())) {
        // 
        DictType dictType = null;
        if (EComponentType.SPARK.getTypeCode().equals(addComponent.getComponentTypeCode())) {
            dictType = DictType.SPARK_VERSION;
        } else if (EComponentType.FLINK.getTypeCode().equals(addComponent.getComponentTypeCode())) {
            dictType = DictType.FLINK_VERSION;
        }
        if (null != dictType) {
            Dict dict = scheduleDictService.getByNameAndValue(dictType.getType(), versionName, null, null);
            if (null != dict) {
                componentVersion = dict.getDictValue();
            }
        }
    }
    // 更新数据库kerberos信息
    KerberosConfig kerberosConfig = consoleKerberosMapper.getByComponentType(clusterId, addComponent.getComponentTypeCode(), componentVersion);
    boolean isFirstOpenKerberos = false;
    if (Objects.isNull(kerberosConfig)) {
        kerberosConfig = new KerberosConfig();
        kerberosConfig.setComponentVersion(componentVersion);
        isFirstOpenKerberos = true;
    }
    kerberosConfig.setOpenKerberos(1);
    kerberosConfig.setRemotePath(remoteDirKerberos);
    kerberosConfig.setClusterId(clusterId);
    kerberosConfig.setComponentType(addComponent.getComponentTypeCode());
    if (keyTabFile != null) {
        kerberosConfig.setName(keyTabFile.getName());
    }
    if (krb5ConfFile != null) {
        kerberosConfig.setKrbName(krb5ConfFile.getName());
    }
    if (StringUtils.isNotEmpty(principal)) {
        kerberosConfig.setPrincipal(principal);
    }
    if (StringUtils.isNotEmpty(principals)) {
        kerberosConfig.setPrincipals(principals);
    }
    if (isFirstOpenKerberos) {
        consoleKerberosMapper.insert(kerberosConfig);
    } else {
        consoleKerberosMapper.updateById(kerberosConfig);
    }
    return remoteDirKerberos;
}

Example 4 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class ConsoleComponentService method downloadFile.

/**
 * 下载文件
 *
 * @param componentId
 * @param downloadType 0:kerberos配置文件 1:配置文件 2:模板文件
 * @return
 */
public File downloadFile(Long componentId, Integer downloadType, Integer componentType, String versionName, Long clusterId, Integer deployType) {
    String localDownLoadPath = "";
    String uploadFileName = "";
    if (null == componentId) {
        EComponentType type = EComponentType.getByCode(componentType);
        // 解析模版中的信息 作为默认值 返回json
        List<ClientTemplate> clientTemplates = this.loadTemplate(clusterId, type, versionName, null, deployType);
        if (CollectionUtils.isNotEmpty(clientTemplates)) {
            Map<String, Object> fileMap = ComponentConfigUtils.convertClientTemplateToMap(clientTemplates);
            uploadFileName = EComponentType.getByCode(componentType).name() + ".json";
            localDownLoadPath = USER_DIR_DOWNLOAD + File.separator + uploadFileName;
            try {
                FileUtils.write(new File(localDownLoadPath), JSONObject.toJSONString(fileMap));
            } catch (Exception e) {
                throw new RdosDefineException("file does not exist");
            }
        }
    } else {
        Component component = componentMapper.selectById(componentId);
        if (null == component) {
            throw new RdosDefineException("Component does not exist");
        }
        SftpConfig sftpConfig = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, SftpConfig.class, null);
        if (null == sftpConfig) {
            throw new RdosDefineException("sftp component does not exist");
        }
        localDownLoadPath = USER_DIR_DOWNLOAD + File.separator + component.getComponentName();
        String remoteDir = sftpConfig.getPath() + File.separator + this.buildSftpPath(clusterId, component.getComponentTypeCode());
        SftpFileManage sftpFileManage = null;
        if (DownloadType.Kerberos.getCode() == downloadType) {
            remoteDir = remoteDir + File.separator + KERBEROS;
            localDownLoadPath = localDownLoadPath + File.separator + KERBEROS;
            sftpFileManage = SftpFileManage.getSftpManager(sftpConfig);
            sftpFileManage.downloadDir(remoteDir, localDownLoadPath);
        } else {
            if (StringUtils.isBlank(component.getUploadFileName())) {
                // 一种是  全部手动填写的 如flink
                EComponentType type = EComponentType.getByCode(componentType);
                String componentConfig = componentService.getComponentByClusterId(clusterId, type.getTypeCode(), true, String.class, component.getVersionValue());
                try {
                    localDownLoadPath = localDownLoadPath + ".json";
                    FileUtils.write(new File(localDownLoadPath), filterConfigMessage(componentConfig));
                } catch (IOException e) {
                    LOGGER.error("write upload file {} error", componentConfig, e);
                }
            } else {
                sftpFileManage = SftpFileManage.getSftpManager(sftpConfig);
                // 一种是 上传配置文件的需要到sftp下载
                sftpFileManage.downloadDir(remoteDir + File.separator + component.getUploadFileName(), localDownLoadPath);
            }
        }
        uploadFileName = component.getUploadFileName();
    }
    File file = new File(localDownLoadPath);
    if (!file.exists()) {
        throw new RdosDefineException("file does not exist");
    }
    String zipFilename = StringUtils.isBlank(uploadFileName) ? "download.zip" : uploadFileName;
    if (file.isDirectory()) {
        // 将文件夹压缩成zip文件
        return zipFile(componentId, downloadType, componentType, file, zipFilename);
    } else {
        return new File(localDownLoadPath);
    }
}

Example 5 with SftpFileManage

use of com.dtstack.taier.pluginapi.sftp.SftpFileManage in project Taier by DTStack.

the class ConsoleComponentService method uploadKerberos.

@Transactional(rollbackFor = Exception.class)
public String uploadKerberos(List<Resource> resources, Long clusterId, Integer componentCode, String versionName) {
    if (CollectionUtils.isEmpty(resources)) {
        throw new RdosDefineException("Please upload a kerberos file!");
    }
    Resource resource = resources.get(0);
    String kerberosFileName = resource.getFileName();
    if (!kerberosFileName.endsWith(ZIP_SUFFIX)) {
        throw new RdosDefineException("Kerberos upload files are not in zip format");
    }
    String sftpComponent = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, String.class, null);
    SftpConfig sftpConfig = getSFTPConfig(sftpComponent, componentCode, "");
    SftpFileManage sftpFileManage = sftpFileManageBean.retrieveSftpManager(sftpConfig);
    String remoteDir = sftpConfig.getPath() + File.separator + this.buildSftpPath(clusterId, componentCode);
    Component addComponent = new Component();
    addComponent.setComponentTypeCode(componentCode);
    addComponent.setVersionName(versionName);
    updateComponentKerberosFile(clusterId, addComponent, sftpFileManage, remoteDir, resource, null, null);
    List<KerberosConfig> kerberosConfigs = consoleKerberosMapper.listAll();
    return mergeKrb5(kerberosConfigs);
}