Examples with FilesystemManager com.dtstack.taier.base.filesystem.FilesystemManager used on opensource projects

Example 1 with FilesystemManager

use of com.dtstack.taier.base.filesystem.FilesystemManager in project Taier by DTStack.

the class SparkYarnClient method init.

@Override
public void init(Properties prop) throws Exception {
    this.sparkExtProp = prop;
    String propStr = PublicUtil.objToString(prop);
    sparkYarnConfig = PublicUtil.jsonStrToObject(propStr, SparkYarnConfig.class);
    setHadoopUserName(sparkYarnConfig);
    initYarnConf(sparkYarnConfig);
    sparkYarnConfig.setDefaultFs(yarnConf.get(HadoopConfTool.FS_DEFAULTFS));
    System.setProperty(SPARK_YARN_MODE, "true");
    parseWebAppAddr();
    logger.info("UGI info: " + UserGroupInformation.getCurrentUser());
    yarnClient = this.buildYarnClient();
    this.filesystemManager = new FilesystemManager(yarnConf, sparkYarnConfig.getSftpConf());
    if (sparkYarnConfig.getMonitorAcceptedApp()) {
        AcceptedApplicationMonitor.start(yarnConf, sparkYarnConfig.getQueue(), sparkYarnConfig);
    }
    this.threadPoolExecutor = new ThreadPoolExecutor(sparkYarnConfig.getAsyncCheckYarnClientThreadNum(), sparkYarnConfig.getAsyncCheckYarnClientThreadNum(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new CustomThreadFactory("spark_yarnclient"));
}

Example 2 with FilesystemManager

use of com.dtstack.taier.base.filesystem.FilesystemManager in project Taier by DTStack.

the class FlinkClient method init.

@Override
public void init(Properties prop) throws Exception {
    this.flinkExtProp = prop;
    String propStr = PublicUtil.objToString(prop);
    flinkConfig = PublicUtil.jsonStrToObject(propStr, FlinkConfig.class);
    syncPluginInfo = SyncPluginInfo.create(flinkConfig);
    sqlPluginInfo = SqlPluginInfo.create(flinkConfig);
    hadoopConf = FlinkClientBuilder.initHadoopConf(flinkConfig);
    flinkClientBuilder = new FlinkClientBuilder(flinkConfig, hadoopConf.getConfiguration(), hadoopConf.getYarnConfiguration());
    flinkClientBuilder.initFlinkGlobalConfiguration(flinkExtProp);
    flinkClusterClientManager = new FlinkClusterClientManager(flinkClientBuilder);
    filesystemManager = new FilesystemManager(hadoopConf.getYarnConfiguration(), flinkConfig.getSftpConf());
    if (flinkConfig.getMonitorAcceptedApp()) {
        AcceptedApplicationMonitor.start(hadoopConf.getYarnConfiguration(), flinkConfig.getQueue(), flinkConfig);
    }
}