Examples with HdfsSourceDTO com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO used on opensource projects

Example 21 with HdfsSourceDTO

use of com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO in project Taier by DTStack.

the class HdfsOperator method getHdfsFileClient.

public static IHdfsFile getHdfsFileClient(Map<String, Object> conf, Map<String, Object> kerberos) {
    HdfsSourceDTO sourceDTO = getSourceDTO(conf, kerberos);
    IHdfsFile hdfsClient = ClientCache.getHdfs(DataSourceType.HDFS.getVal());
    return hdfsClient;
}

Example 22 with HdfsSourceDTO

use of com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO in project Taier by DTStack.

the class HdfsOperator method copyToLocal.

public static void copyToLocal(Map<String, Object> conf, Map<String, Object> kerberos, String srcPath, String dstPath) {
    HdfsSourceDTO sourceDTO = getSourceDTO(conf, kerberos);
    IHdfsFile hdfsClient = ClientCache.getHdfs(DataSourceType.HDFS.getVal());
    hdfsClient.copyToLocal(sourceDTO, srcPath, dstPath);
}

Example 23 with HdfsSourceDTO

use of com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO in project Taier by DTStack.

the class HdfsOperator method copyFromLocal.

public static void copyFromLocal(Map<String, Object> conf, Map<String, Object> kerberos, String srcPath, String dstPath, boolean overwrite) {
    HdfsSourceDTO sourceDTO = getSourceDTO(conf, kerberos);
    IHdfsFile hdfsClient = ClientCache.getHdfs(DataSourceType.HDFS.getVal());
    hdfsClient.copyFromLocal(sourceDTO, srcPath, dstPath, overwrite);
}

Example 24 with HdfsSourceDTO

use of com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO in project Taier by DTStack.

the class HdfsOperator method getContentSummary.

/**
 * 获取hdfs上指定路径的ContentSummary
 *
 * @param conf
 * @param kerberos
 * @param targetPath
 * @return
 */
public static HDFSContentSummary getContentSummary(Map<String, Object> conf, Map<String, Object> kerberos, String targetPath) {
    try {
        HdfsSourceDTO sourceDTO = getSourceDTO(conf, kerberos);
        IHdfsFile hdfsClient = ClientCache.getHdfs(DataSourceType.HDFS.getVal());
        return hdfsClient.getContentSummary(sourceDTO, targetPath);
    } catch (Exception e) {
        throw new DtCenterDefException(String.format("获取hdfs文件内容摘要异常!原因是：%s", e.getMessage()));
    }
}

Example 25 with HdfsSourceDTO

use of com.dtstack.dtcenter.loader.dto.source.HdfsSourceDTO in project Taier by DTStack.

the class HdfsOperator method copyDirector.

/**
 * 获取hdfs上指定路径集合的ContentSummary
 *
 * @param conf hadoop配置
 * @param kerberos kerberos配置
 * @param src 原路径
 * @param dist 复制路径
 */
public static void copyDirector(Map<String, Object> conf, Map<String, Object> kerberos, String src, String dist) {
    try {
        HdfsSourceDTO sourceDTO = getSourceDTO(conf, kerberos);
        IHdfsFile hdfsClient = ClientCache.getHdfs(DataSourceType.HDFS.getVal());
        hdfsClient.copyDirector(sourceDTO, src, dist);
    } catch (Exception e) {
        throw new DtCenterDefException(String.format("hdfs内复制文件夹异常!原因是：%s", e.getMessage()));
    }
}