Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 86 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class NomadWorkerStarter method loadConfigurations.

private Config loadConfigurations(CommandLine cmd, int id) {
    String twister2Home = cmd.getOptionValue("twister2_home");
    String container = cmd.getOptionValue("container_class");
    String configDir = cmd.getOptionValue("config_dir");
    String clusterType = cmd.getOptionValue("cluster_type");
    String jobID = cmd.getOptionValue("job_id");
    LOG.log(Level.FINE, String.format("Initializing process with " + "twister_home: %s container_class: %s config_dir: %s cluster_type: %s", twister2Home, container, configDir, clusterType));
    Config cfg = ConfigLoader.loadConfig(twister2Home, configDir, clusterType);
    Config workerConfig = Config.newBuilder().putAll(cfg).put(SchedulerContext.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.WORKER_CLASS, container).put(SchedulerContext.TWISTER2_CONTAINER_ID, id).put(SchedulerContext.TWISTER2_CLUSTER_TYPE, clusterType).build();
    String jobDescFile = JobUtils.getJobDescriptionFilePath(jobID, workerConfig);
    job = JobUtils.readJobFile(jobDescFile);
    job.getNumberOfWorkers();
    Config updatedConfig = JobUtils.overrideConfigs(job, cfg);
    updatedConfig = Config.newBuilder().putAll(updatedConfig).put(SchedulerContext.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.WORKER_CLASS, container).put(SchedulerContext.TWISTER2_CONTAINER_ID, id).put(SchedulerContext.TWISTER2_CLUSTER_TYPE, clusterType).put(SchedulerContext.JOB_ID, job.getJobId()).put(SchedulerContext.JOB_NAME, job.getJobName()).build();
    return updatedConfig;
}

Example 87 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class MPIWorkerStarter method loadConfigurations.

private Config loadConfigurations(CommandLine cmd) {
    String twister2Home = cmd.getOptionValue("twister2_home");
    String configDir = cmd.getOptionValue("config_dir");
    String clusterType = cmd.getOptionValue("cluster_type");
    String jobId = cmd.getOptionValue("job_id");
    String jIp = cmd.getOptionValue("job_master_ip");
    int jPort = Integer.parseInt(cmd.getOptionValue("job_master_port"));
    boolean restoreJob = Boolean.parseBoolean(cmd.getOptionValue("restore_job"));
    restartCount = Integer.parseInt(cmd.getOptionValue("restart_count"));
    Config cfg = ConfigLoader.loadConfig(twister2Home, configDir, clusterType);
    Config workerConfig = Config.newBuilder().putAll(cfg).put(MPIContext.TWISTER2_HOME.getKey(), twister2Home).put(MPIContext.TWISTER2_CONTAINER_ID, globalRank).put(MPIContext.TWISTER2_CLUSTER_TYPE, clusterType).build();
    String jobDescFile = JobUtils.getJobDescriptionFilePath(jobId, workerConfig);
    job = JobUtils.readJobFile(jobDescFile);
    Config updatedConfig = JobUtils.overrideConfigs(job, cfg);
    updatedConfig = Config.newBuilder().putAll(updatedConfig).put(MPIContext.TWISTER2_HOME.getKey(), twister2Home).put(MPIContext.WORKER_CLASS, job.getWorkerClassName()).put(MPIContext.TWISTER2_CONTAINER_ID, globalRank).put(MPIContext.JOB_ID, jobId).put(MPIContext.JOB_OBJECT, job).put(MPIContext.TWISTER2_CLUSTER_TYPE, clusterType).put(JobMasterContext.JOB_MASTER_IP, jIp).put(JobMasterContext.JOB_MASTER_PORT, jPort).put(ZKContext.SERVER_ADDRESSES, null).put(CheckpointingContext.CHECKPOINTING_RESTORE_JOB, restoreJob).build();
    LOG.log(Level.FINE, String.format("Initializing process with " + "twister_home: %s worker_class: %s config_dir: %s cluster_type: %s", twister2Home, job.getWorkerClassName(), configDir, clusterType));
    return updatedConfig;
}

Example 88 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class NomadJobMasterStarter method loadConfigurations.

private Config loadConfigurations(CommandLine cmd, int id) {
    String twister2Home = cmd.getOptionValue("twister2_home");
    String container = cmd.getOptionValue("container_class");
    String configDir = cmd.getOptionValue("config_dir");
    String clusterType = cmd.getOptionValue("cluster_type");
    String jobId = cmd.getOptionValue("job_id");
    LOG.log(Level.FINE, String.format("Initializing process with " + "twister_home: %s container_class: %s config_dir: %s cluster_type: %s", twister2Home, container, configDir, clusterType));
    Config cfg = ConfigLoader.loadConfig(twister2Home, configDir, clusterType);
    Config workerConfig = Config.newBuilder().putAll(cfg).put(SchedulerContext.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.WORKER_CLASS, container).put(SchedulerContext.TWISTER2_CONTAINER_ID, id).put(SchedulerContext.JOB_ID, jobId).put(SchedulerContext.TWISTER2_CLUSTER_TYPE, clusterType).build();
    String jobDescFile = JobUtils.getJobDescriptionFilePath(jobId, workerConfig);
    job = JobUtils.readJobFile(jobDescFile);
    job.getNumberOfWorkers();
    Config updatedConfig = JobUtils.overrideConfigs(job, cfg);
    updatedConfig = Config.newBuilder().putAll(updatedConfig).put(SchedulerContext.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.WORKER_CLASS, container).put(SchedulerContext.TWISTER2_CONTAINER_ID, id).put(SchedulerContext.TWISTER2_CLUSTER_TYPE, clusterType).put(SchedulerContext.JOB_ID, jobId).put(SchedulerContext.JOB_NAME, job.getJobName()).build();
    return updatedConfig;
}

Example 89 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class LFSTest method main.

public static void main(String[] args) throws IOException {
    Config config = Config.newBuilder().put(FsContext.UPLOAD_DIRECTORY, "/home/user/Desktop/test1/").build();
    LocalFileSystemUploader lsf = new LocalFileSystemUploader();
    lsf.initialize(config, null);
    URI returned = lsf.uploadPackage("/home/user/Desktop/tobecopied/");
    System.out.printf(returned.toString());
}

Example 90 with Config

use of edu.iu.dsc.tws.api.config.Config in project beam by apache.

the class BeamBatchWorker method execute.

@Override
public void execute(BatchTSetEnvironment env) {
    Config config = env.getConfig();
    Map<String, String> sideInputIds = (LinkedHashMap<String, String>) config.get(SIDEINPUTS);
    Set<String> leaveIds = (Set<String>) config.get(LEAVES);
    TBaseGraph graph = (TBaseGraph) config.get(GRAPH);
    env.settBaseGraph(graph);
    setupTSets(env, sideInputIds, leaveIds);
    resetEnv(env, graph);
    executePipeline(env);
}