Examples with Twister2Job edu.iu.dsc.tws.api.Twister2Job used on opensource projects

Example 1 with Twister2Job

use of edu.iu.dsc.tws.api.Twister2Job in project twister2 by DSC-SPIDAL.

the class PytorchSampleJob method main.

public static void main(String[] args) throws RuntimeException {
    scriptPath = "";
    workerProcessInstances = 2;
    masterProcessInstances = 2;
    pythonPath = "";
    if (args.length < 3) {
        throw new RuntimeException("Need the following parameters: " + "--scriptPath <path to bash script which runs the programme> " + "--masterInstances <processes for master programme> " + "--workerInstances <number of child processes to be spawned> " + "--pythonPath <path to python executable> ");
    } else {
        scriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        workerProcessInstances = ParameterTool.fromArgs(args).getInt("masterInstances");
        workerProcessInstances = ParameterTool.fromArgs(args).getInt("workerInstances");
        pythonPath = ParameterTool.fromArgs(args).get("pythonPath");
    }
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Hello");
    jobConfig.put("scriptPath", scriptPath);
    jobConfig.put("workerProcessInstances", workerProcessInstances);
    jobConfig.put("masterProcessInstances", masterProcessInstances);
    jobConfig.put("pythonPath", pythonPath);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("twister2-pytorch-sample").setWorkerClass(PytorchSampleJob.class).addComputeResource(2, 1024, masterProcessInstances).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 2 with Twister2Job

use of edu.iu.dsc.tws.api.Twister2Job in project twister2 by DSC-SPIDAL.

the class HelloTwister2Pytorch method main.

public static void main(String[] args) {
    // lets take number of workers as an command line argument
    String localScriptPath = "";
    int localWorkers = 2;
    System.out.println("---------------------------");
    System.out.println(Arrays.toString(args));
    if (args.length < 2) {
        throw new RuntimeException("Invalid number of arguments");
    } else {
        localScriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        localWorkers = ParameterTool.fromArgs(args).getInt("workers");
    }
    // String path = ParameterTool.fromArgs(args).get("scriptPath");
    // System.out.println("Script Path : " + path);
    System.out.println("---------------------------");
    int numberOfWorkers = 2;
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Spawn-Hello");
    jobConfig.put("scriptPath", localScriptPath);
    jobConfig.put("workers", localWorkers);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("hello-world-job-spawn").setWorkerClass(HelloTwister2Pytorch.class).addComputeResource(2, 1024, numberOfWorkers).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 3 with Twister2Job

use of edu.iu.dsc.tws.api.Twister2Job in project twister2 by DSC-SPIDAL.

the class Twister2Mnist method main.

public static void main(String[] args) {
    // lets take number of workers as an command line argument
    String localScriptPath = "";
    int localWorkers = 2;
    System.out.println("---------------------------");
    System.out.println(Arrays.toString(args));
    if (args.length < 2) {
        throw new RuntimeException("Invalid number of arguments");
    } else {
        localScriptPath = ParameterTool.fromArgs(args).get("scriptPath");
        localWorkers = ParameterTool.fromArgs(args).getInt("workers");
    }
    // String path = ParameterTool.fromArgs(args).get("scriptPath");
    // System.out.println("Script Path : " + path);
    System.out.println("---------------------------");
    int numberOfWorkers = 2;
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // lets put a configuration here
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("hello-key", "Twister2-Spawn-Hello");
    jobConfig.put("scriptPath", localScriptPath);
    jobConfig.put("workers", localWorkers);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("hello-world-job-spawn").setWorkerClass(Twister2Mnist.class).addComputeResource(2, 1024, numberOfWorkers).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}

Example 4 with Twister2Job

use of edu.iu.dsc.tws.api.Twister2Job in project twister2 by DSC-SPIDAL.

the class BasicAuroraJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    System.out.println("read config values: " + config.size());
    System.out.println(config);
    // build JobConfig
    HashMap<String, Object> configurations = new HashMap<>();
    configurations.put(SchedulerContext.THREADS_PER_WORKER, 8);
    JobConfig jobConfig = new JobConfig();
    jobConfig.putAll(configurations);
    // build the job
    Twister2Job twister2Job = Twister2Job.loadTwister2Job(config, jobConfig);
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
    // now terminate the job
    terminateJob(config);
// jobWriteTest(twister2Job);
// jobReadTest();
}

Example 5 with Twister2Job

use of edu.iu.dsc.tws.api.Twister2Job in project twister2 by DSC-SPIDAL.

the class KafkaExample method main.

public static void main(String[] args) throws ParseException {
    Options options = new Options();
    options.addOption(CLI_SERVER, true, "Kafka bootstrap server in the format host:port");
    options.addOption(CLI_TOPICS, true, "Set of topics in the format topic1,topic2");
    CommandLineParser cliParser = new DefaultParser();
    CommandLine cli = cliParser.parse(options, args);
    HashMap<String, Object> configs = new HashMap<>();
    configs.put(CLI_SERVER, "localhost:9092");
    configs.put(CLI_TOPICS, Collections.singleton("test2"));
    if (cli.hasOption(CLI_SERVER)) {
        configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, cli.getOptionValue(CLI_SERVER));
    }
    if (cli.hasOption(CLI_TOPICS)) {
        String topics = cli.getOptionValue(CLI_TOPICS);
        Set<String> topicsSet = Arrays.stream(topics.split(",")).map(String::trim).collect(Collectors.toSet());
        configs.put(CLI_TOPICS, topicsSet);
    }
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    JobConfig jobConfig = new JobConfig();
    jobConfig.putAll(configs);
    Twister2Job twister2Job;
    twister2Job = Twister2Job.newBuilder().setJobName(KafkaExample.class.getName()).setWorkerClass(KafkaExample.class).addComputeResource(1, 1024, 1).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
}