Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 1 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class PytorchSampleJob method main.

public static void main(String[] args) throws RuntimeException {
    scriptPath = "";
    workerProcessInstances = 2;
    masterProcessInstances = 2;
    pythonPath = "";
    if (args.length < 3) {
        throw new RuntimeException("Need the following parameters: " + "--scriptPath <path to bash script which runs the programme> " + "--masterInstances <processes for master programme> " + "--workerInstances <number of child processes to be spawned> " + "--pythonPath <path to python executable> ");
    } else {
        scriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        workerProcessInstances = ParameterTool.fromArgs(args).getInt("masterInstances");
        workerProcessInstances = ParameterTool.fromArgs(args).getInt("workerInstances");
        pythonPath = ParameterTool.fromArgs(args).get("pythonPath");
    }
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Hello");
    jobConfig.put("scriptPath", scriptPath);
    jobConfig.put("workerProcessInstances", workerProcessInstances);
    jobConfig.put("masterProcessInstances", masterProcessInstances);
    jobConfig.put("pythonPath", pythonPath);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("twister2-pytorch-sample").setWorkerClass(PytorchSampleJob.class).addComputeResource(2, 1024, masterProcessInstances).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 2 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class HelloTwister2Pytorch method main.

public static void main(String[] args) {
    // lets take number of workers as an command line argument
    String localScriptPath = "";
    int localWorkers = 2;
    System.out.println("---------------------------");
    System.out.println(Arrays.toString(args));
    if (args.length < 2) {
        throw new RuntimeException("Invalid number of arguments");
    } else {
        localScriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        localWorkers = ParameterTool.fromArgs(args).getInt("workers");
    }
    // String path = ParameterTool.fromArgs(args).get("scriptPath");
    // System.out.println("Script Path : " + path);
    System.out.println("---------------------------");
    int numberOfWorkers = 2;
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Spawn-Hello");
    jobConfig.put("scriptPath", localScriptPath);
    jobConfig.put("workers", localWorkers);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("hello-world-job-spawn").setWorkerClass(HelloTwister2Pytorch.class).addComputeResource(2, 1024, numberOfWorkers).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 3 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class Twister2Mnist method main.

public static void main(String[] args) {
    // lets take number of workers as an command line argument
    String localScriptPath = "";
    int localWorkers = 2;
    System.out.println("---------------------------");
    System.out.println(Arrays.toString(args));
    if (args.length < 2) {
        throw new RuntimeException("Invalid number of arguments");
    } else {
        localScriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        localWorkers = ParameterTool.fromArgs(args).getInt("workers");
    }
    // String path = ParameterTool.fromArgs(args).get("scriptPath");
    // System.out.println("Script Path : " + path);
    System.out.println("---------------------------");
    int numberOfWorkers = 2;
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Spawn-Hello");
    jobConfig.put("scriptPath", localScriptPath);
    jobConfig.put("workers", localWorkers);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("hello-world-job-spawn").setWorkerClass(Twister2Mnist.class).addComputeResource(2, 1024, numberOfWorkers).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 4 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class K8sControllerExample method test1.

/**
 * test method
 */
public static void test1(JobAPI.Job job) {
    String jobID = "ft-job";
    String jobID2 = "ft-job-2";
    Config cnfg = Config.newBuilder().put("nothing", "nothing").build();
    RequestObjectBuilder.init(cnfg, jobID, 0, 0, null);
    V1ConfigMap cm = RequestObjectBuilder.createConfigMap(job);
    RequestObjectBuilder.init(cnfg, jobID2, 0, 0, null);
    V1ConfigMap cm2 = RequestObjectBuilder.createConfigMap(job);
    KubernetesController controller = KubernetesController.init("default");
    String key0 = KubernetesUtils.createRestartWorkerKey(0);
    String key1 = KubernetesUtils.createRestartJobMasterKey();
    if (controller.existConfigMap(cm2.getMetadata().getName())) {
        controller.deleteConfigMap(cm2.getMetadata().getName());
    } else {
        controller.createConfigMap(cm2);
        int restartCount = controller.getRestartCount(jobID2, key0);
        LOG.info("restartCount: " + restartCount);
        controller.addConfigMapParam(jobID2, key0, 0 + "");
        LOG.info("added restartCount: " + controller.getRestartCount(jobID2, key0));
        if (controller.updateConfigMapParam(jobID2, key0, 1 + "")) {
            LOG.info("updated restartCount: " + controller.getRestartCount(jobID2, key0));
        } else {
            LOG.info("Cannot update restartCount");
            LOG.info("restartCount: " + controller.getRestartCount(jobID2, key0));
        }
        if (controller.updateConfigMapParam(jobID2, key1, 10 + "")) {
            LOG.info("updated restartCount: " + controller.getRestartCount(jobID2, key1));
        } else {
            LOG.info("Cannot update restartCount");
            LOG.info("restartCount: " + controller.getRestartCount(jobID2, key1));
        }
        if (controller.removeRestartCount(jobID2, key0)) {
            LOG.info("removed restartCount: " + controller.getRestartCount(jobID2, key0));
        } else {
            LOG.info("cannot remove restartCount: " + controller.getRestartCount(jobID2, key0));
        }
    }
    if (controller.existConfigMap(cm.getMetadata().getName())) {
        controller.deleteConfigMap(cm.getMetadata().getName());
    } else {
        controller.createConfigMap(cm);
        int restartCount = controller.getRestartCount(jobID, key0);
        LOG.info("restartCount: " + restartCount);
    }
    LOG.info("done");
}

Example 5 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class MultiComputeTasksGraphExample method main.

public static void main(String[] args) throws ParseException {
    LOG.log(Level.INFO, "MultiComputeTaskGraph");
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    HashMap<String, Object> configurations = new HashMap<>();
    configurations.put(SchedulerContext.THREADS_PER_WORKER, 8);
    Options options = new Options();
    options.addOption(DataObjectConstants.PARALLELISM_VALUE, true, "parallelism");
    options.addOption(DataObjectConstants.WORKERS, true, "workers");
    options.addOption(DataObjectConstants.DSIZE, true, "dsize");
    // for testing data locality
    options.addOption(DataObjectConstants.DIMENSIONS, true, "dim");
    options.addOption(DataObjectConstants.CSIZE, true, "csize");
    options.addOption(DataObjectConstants.DINPUT_DIRECTORY, true, "dinput");
    options.addOption(DataObjectConstants.CINPUT_DIRECTORY, true, "cinput");
    options.addOption(DataObjectConstants.FILE_SYSTEM, true, "filesys");
    options.addOption(DataObjectConstants.NUMBER_OF_FILES, true, "nFiles");
    CommandLineParser commandLineParser = new DefaultParser();
    CommandLine cmd = commandLineParser.parse(options, args);
    int workers = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.WORKERS));
    int parallelismValue = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.PARALLELISM_VALUE));
    int dsize = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.DSIZE));
    int csize = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.CSIZE));
    int dimension = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.DIMENSIONS));
    int numberOfFiles = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.NUMBER_OF_FILES));
    String filesys = cmd.getOptionValue(DataObjectConstants.FILE_SYSTEM);
    String dinput = cmd.getOptionValue(DataObjectConstants.DINPUT_DIRECTORY);
    String cinput = cmd.getOptionValue(DataObjectConstants.CINPUT_DIRECTORY);
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    jobConfig.put(DataObjectConstants.WORKERS, Integer.toString(workers));
    jobConfig.put(DataObjectConstants.PARALLELISM_VALUE, Integer.toString(parallelismValue));
    jobConfig.put(DataObjectConstants.DSIZE, Integer.toString(dsize));
    jobConfig.put(DataObjectConstants.CSIZE, Integer.toString(csize));
    jobConfig.put(DataObjectConstants.DINPUT_DIRECTORY, dinput);
    jobConfig.put(DataObjectConstants.CINPUT_DIRECTORY, cinput);
    jobConfig.put(DataObjectConstants.FILE_SYSTEM, filesys);
    jobConfig.put(DataObjectConstants.NUMBER_OF_FILES, Integer.toString(numberOfFiles));
    jobConfig.put(DataObjectConstants.DIMENSIONS, Integer.toString(dimension));
    jobConfig.putAll(configurations);
    // build the job
    Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
    jobBuilder.setJobName("MultiComputeTasksGraph");
    jobBuilder.setWorkerClass(MultiComputeTasksGraphExample.class.getName());
    jobBuilder.addComputeResource(2, 512, 1.0, workers);
    jobBuilder.setConfig(jobConfig);
    // now submit the job
    Twister2Submitter.submitJob(jobBuilder.build(), config);
}