Examples with Resource com.dtstack.taier.dao.dto.Resource used on opensource projects

Example 1 with Resource

use of com.dtstack.taier.dao.dto.Resource in project Taier by DTStack.

the class ConsoleComponentService method uploadResourceToSftp.

private String uploadResourceToSftp(Long clusterId, List<Resource> resources, String kerberosFileName, SftpConfig sftpConfig, Component addComponent, Component dbComponent, String principals, String principal) {
    // 上传配置文件到sftp 供后续下载
    SftpFileManage sftpFileManage = sftpFileManageBean.retrieveSftpManager(sftpConfig);
    String md5sum = "";
    String remoteDir = sftpConfig.getPath() + File.separator + this.buildSftpPath(clusterId, addComponent.getComponentTypeCode());
    for (Resource resource : resources) {
        if (!resource.getFileName().equalsIgnoreCase(kerberosFileName) || StringUtils.isBlank(kerberosFileName)) {
            addComponent.setUploadFileName(resource.getFileName());
        }
        try {
            if (resource.getFileName().equalsIgnoreCase(kerberosFileName)) {
                // 更新Kerberos信息
                this.updateComponentKerberosFile(clusterId, addComponent, sftpFileManage, remoteDir, resource, principals, principal);
            } else {
                LOGGER.info("start upload hadoop config file:{}", kerberosFileName);
                this.updateComponentConfigFile(dbComponent, sftpFileManage, remoteDir, resource);
                if (EComponentType.HDFS.getTypeCode().equals(addComponent.getComponentTypeCode())) {
                    String xmlZipLocation = resource.getUploadedFileName();
                    md5sum = MD5Util.getFileMd5String(new File(xmlZipLocation));
                    this.updateConfigToSftpPath(clusterId, sftpConfig, sftpFileManage, resource, null, addComponent.getComponentTypeCode());
                }
                if (EComponentType.YARN.getTypeCode().equals(addComponent.getComponentTypeCode())) {
                    List<ComponentConfig> clientTemplates = scheduleDictService.loadExtraComponentConfig(addComponent.getVersionValue(), addComponent.getComponentTypeCode());
                    this.updateConfigToSftpPath(clusterId, sftpConfig, sftpFileManage, resource, clientTemplates, addComponent.getComponentTypeCode());
                }
            }
        } catch (Exception e) {
            LOGGER.error("update component resource {}  error", resource.getUploadedFileName(), e);
            if (e instanceof RdosDefineException) {
                throw (RdosDefineException) e;
            } else {
                throw new RdosDefineException("Failed to update component");
            }
        } finally {
            try {
                FileUtils.forceDelete(new File(resource.getUploadedFileName()));
            } catch (IOException e) {
                LOGGER.error("delete upload file {} error", resource.getUploadedFileName(), e);
            }
        }
    }
    return md5sum;
}

Example 2 with Resource

use of com.dtstack.taier.dao.dto.Resource in project Taier by DTStack.

the class ConsoleComponentService method updateComponentKerberosFile.

/**
 * 解压kerberos文件到本地 并上传至sftp
 * * @param clusterId
 *
 * @param addComponent
 * @param remoteDir
 * @param resource
 * @return
 */
private String updateComponentKerberosFile(Long clusterId, Component addComponent, SftpFileManage sftpFileManage, String remoteDir, Resource resource, String principals, String principal) {
    File keyTabFile = null;
    File krb5ConfFile = null;
    String remoteDirKerberos = remoteDir + File.separator + KERBEROS;
    if (resource != null) {
        // kerberos认证文件 远程删除 kerberos下的文件
        LOGGER.info("updateComponentKerberosFile remote path:{}", remoteDirKerberos);
        // 删除本地文件夹
        String kerberosPath = this.getLocalKerberosPath(clusterId, addComponent.getComponentTypeCode());
        try {
            FileUtils.deleteDirectory(new File(kerberosPath));
        } catch (IOException e) {
            LOGGER.error("delete old kerberos directory {} error", kerberosPath, e);
        }
        // 解压到本地
        List<File> files = ZipUtil.upzipFile(resource.getUploadedFileName(), kerberosPath);
        if (CollectionUtils.isEmpty(files)) {
            throw new RdosDefineException("Hadoop-Kerberos file decompression error");
        }
        keyTabFile = files.stream().filter(f -> f.getName().endsWith(KEYTAB_SUFFIX)).findFirst().orElse(null);
        krb5ConfFile = files.stream().filter(f -> f.getName().equalsIgnoreCase(KRB5_CONF)).findFirst().orElse(null);
        if (keyTabFile == null) {
            throw new RdosDefineException("There must be a keytab file in the zip file of the uploaded Hadoop-Kerberos file, please add the keytab file");
        }
        LOGGER.info("fileKeyTab Unzip fileName:{}", keyTabFile.getAbsolutePath());
        if (krb5ConfFile == null) {
            throw new RdosDefineException("There must be a krb5.conf file in the zip file of the uploaded Hadoop-Kerberos file, please add the krb5.conf file");
        }
        LOGGER.info("conf Unzip fileName:{}", krb5ConfFile.getAbsolutePath());
        // 获取principal
        List<PrincipalName> principalLists = this.getPrincipal(keyTabFile);
        principal = parsePrincipal(principal, principalLists);
        if (StringUtils.isEmpty(principals)) {
            List<String> principalNames = new ArrayList<>();
            for (PrincipalName principalName : principalLists) {
                principalNames.add(principalName.getName());
            }
            principals = StringUtils.join(principalNames, ",");
        }
        // 删除sftp原来kerberos 的文件夹
        sftpFileManage.deleteDir(remoteDirKerberos);
        // 上传kerberos解压后的文件
        for (File file : files) {
            LOGGER.info("upload sftp file:{}", file.getAbsolutePath());
            sftpFileManage.uploadFile(remoteDirKerberos, file.getPath());
        }
    }
    String versionName = addComponent.getVersionName();
    String componentVersion = "";
    if (StringUtils.isNotBlank(versionName) && ComponentVersionUtil.isMultiVersionComponent(addComponent.getComponentTypeCode())) {
        // 
        DictType dictType = null;
        if (EComponentType.SPARK.getTypeCode().equals(addComponent.getComponentTypeCode())) {
            dictType = DictType.SPARK_VERSION;
        } else if (EComponentType.FLINK.getTypeCode().equals(addComponent.getComponentTypeCode())) {
            dictType = DictType.FLINK_VERSION;
        }
        if (null != dictType) {
            Dict dict = scheduleDictService.getByNameAndValue(dictType.getType(), versionName, null, null);
            if (null != dict) {
                componentVersion = dict.getDictValue();
            }
        }
    }
    // 更新数据库kerberos信息
    KerberosConfig kerberosConfig = consoleKerberosMapper.getByComponentType(clusterId, addComponent.getComponentTypeCode(), componentVersion);
    boolean isFirstOpenKerberos = false;
    if (Objects.isNull(kerberosConfig)) {
        kerberosConfig = new KerberosConfig();
        kerberosConfig.setComponentVersion(componentVersion);
        isFirstOpenKerberos = true;
    }
    kerberosConfig.setOpenKerberos(1);
    kerberosConfig.setRemotePath(remoteDirKerberos);
    kerberosConfig.setClusterId(clusterId);
    kerberosConfig.setComponentType(addComponent.getComponentTypeCode());
    if (keyTabFile != null) {
        kerberosConfig.setName(keyTabFile.getName());
    }
    if (krb5ConfFile != null) {
        kerberosConfig.setKrbName(krb5ConfFile.getName());
    }
    if (StringUtils.isNotEmpty(principal)) {
        kerberosConfig.setPrincipal(principal);
    }
    if (StringUtils.isNotEmpty(principals)) {
        kerberosConfig.setPrincipals(principals);
    }
    if (isFirstOpenKerberos) {
        consoleKerberosMapper.insert(kerberosConfig);
    } else {
        consoleKerberosMapper.updateById(kerberosConfig);
    }
    return remoteDirKerberos;
}

Example 3 with Resource

use of com.dtstack.taier.dao.dto.Resource in project Taier by DTStack.

the class ConsoleComponentService method uploadKerberos.

@Transactional(rollbackFor = Exception.class)
public String uploadKerberos(List<Resource> resources, Long clusterId, Integer componentCode, String versionName) {
    if (CollectionUtils.isEmpty(resources)) {
        throw new RdosDefineException("Please upload a kerberos file!");
    }
    Resource resource = resources.get(0);
    String kerberosFileName = resource.getFileName();
    if (!kerberosFileName.endsWith(ZIP_SUFFIX)) {
        throw new RdosDefineException("Kerberos upload files are not in zip format");
    }
    String sftpComponent = componentService.getComponentByClusterId(clusterId, EComponentType.SFTP.getTypeCode(), false, String.class, null);
    SftpConfig sftpConfig = getSFTPConfig(sftpComponent, componentCode, "");
    SftpFileManage sftpFileManage = sftpFileManageBean.retrieveSftpManager(sftpConfig);
    String remoteDir = sftpConfig.getPath() + File.separator + this.buildSftpPath(clusterId, componentCode);
    Component addComponent = new Component();
    addComponent.setComponentTypeCode(componentCode);
    addComponent.setVersionName(versionName);
    updateComponentKerberosFile(clusterId, addComponent, sftpFileManage, remoteDir, resource, null, null);
    List<KerberosConfig> kerberosConfigs = consoleKerberosMapper.listAll();
    return mergeKrb5(kerberosConfigs);
}

Example 4 with Resource

use of com.dtstack.taier.dao.dto.Resource in project Taier by DTStack.

the class UploadController method addOrUpdateComponent.

@PostMapping(value = "/component/addOrUpdateComponent")
public R<ComponentVO> addOrUpdateComponent(@RequestParam("resources1") List<MultipartFile> files1, @RequestParam("resources2") List<MultipartFile> files2, @RequestParam("clusterId") Long clusterId, @RequestParam(value = "componentConfig") String componentConfig, @RequestParam("versionName") @NotNull String versionName, @RequestParam("kerberosFileName") String kerberosFileName, @RequestParam("componentTemplate") String componentTemplate, @RequestParam("componentCode") Integer componentCode, @RequestParam("storeType") Integer storeType, @RequestParam("principals") String principals, @RequestParam("principal") String principal, @RequestParam(value = "isMetadata", defaultValue = "false") Boolean isMetadata, @RequestParam(value = "isDefault", defaultValue = "false") Boolean isDefault, @RequestParam(value = "deployType") Integer deployType) {
    List<Resource> resources = getResourcesFromFiles(files1);
    List<Resource> resourcesAdd = getResourcesFromFiles(files2);
    resources.addAll(resourcesAdd);
    return new APITemplate<ComponentVO>() {

        @Override
        protected void checkParams() throws IllegalArgumentException {
            if (null == componentCode) {
                throw new RdosDefineException("Component type cannot be empty");
            }
            if (null == clusterId) {
                throw new RdosDefineException("Cluster Id cannot be empty");
            }
            if (CollectionUtils.isNotEmpty(resources) && resources.size() >= 2 && StringUtils.isBlank(kerberosFileName)) {
                // 上传二份文件 需要kerberosFileName文件名字段
                throw new RdosDefineException("kerberosFileName不能为空");
            }
            // 校验引擎是否添加
            if (EComponentType.deployTypeComponents.contains(componentCode) && null == deployType) {
                throw new RdosDefineException(ErrorCode.EMPTY_PARAMETERS.getMsg() + ":deployType");
            }
        }

        @Override
        protected ComponentVO process() throws RdosDefineException {
            // 校验引擎是否添加
            EComponentType componentType = EComponentType.getByCode(componentCode);
            if (EComponentType.deployTypeComponents.contains(componentType) && null == deployType) {
                throw new RdosDefineException("deploy type cannot be empty");
            }
            return consoleComponentService.addOrUpdateComponent(clusterId, componentConfig, resources, versionName, kerberosFileName, componentTemplate, componentType, storeType, principals, principal, isMetadata, isDefault, deployType);
        }
    }.execute();
}

Example 5 with Resource

use of com.dtstack.taier.dao.dto.Resource in project Taier by DTStack.

the class UploadController method getResourcesFromFiles.

private List<Resource> getResourcesFromFiles(List<MultipartFile> files) {
    List<Resource> resources = new ArrayList<>(files.size());
    for (MultipartFile file : files) {
        String fileOriginalName = file.getOriginalFilename();
        String path = uploadsDir + File.separator + fileOriginalName;
        File saveFile = new File(path);
        if (!saveFile.getParentFile().exists()) {
            saveFile.getParentFile().mkdirs();
        }
        try {
            file.transferTo(saveFile);
        } catch (Exception e) {
            LOGGER.error("", e);
            throw new RdosDefineException("An error occurred while storing the file");
        }
        resources.add(new Resource(fileOriginalName, path, (int) file.getSize(), file.getContentType(), file.getName()));
    }
    return resources;
}