Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 96 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class BasicKubernetesJob method main.

@SuppressWarnings("unchecked")
public static void main(String[] args) {
    // LoggingHelper.setupLogging(null, "logs", "client");
    LOG.info("Job submission time: " + System.currentTimeMillis());
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    LOG.fine("read config values: " + config.size() + "\n" + config);
    submitJob(config);
}

Example 97 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class BasicMesosJob method main.

public static void main(String[] args) {
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    System.out.println("read config values: " + config.size());
    System.out.println(config);
    String jobName = SchedulerContext.jobName(config);
    jobName += "-" + System.currentTimeMillis();
    System.out.println("job name is " + jobName);
    // build JobConfig
    HashMap<String, Object> configurations = new HashMap<>();
    configurations.put(SchedulerContext.THREADS_PER_WORKER, 8);
    JobConfig jobConfig = new JobConfig();
    jobConfig.putAll(configurations);
    // build the job
    Twister2Job twister2Job = Twister2Job.loadTwister2Job(config, jobConfig);
    twister2Job.setJobName(jobName);
    // now submit the job
    Twister2Submitter.submitJob(twister2Job, config);
    System.out.println("now terminating...");
    Twister2Submitter.terminateJob(twister2Job.getJobName(), config);
}

Example 98 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class ZKJobMasterFinderExample method main.

/**
 * This class is used together with ZKJobMasterRegistrarExample.java
 * That class registers the Job Master and this class discovers it
 * <p>
 * This class tries to get the Job Master address from a ZooKeeper server
 * If the Job Master has not been registered yet,
 * it can wait for it to be registered
 */
public static void main(String[] args) {
    if (args.length != 1) {
        printUsage();
        return;
    }
    String zkAddress = args[0];
    String jobID = "test-job";
    Config cnfg = buildTestConfig(zkAddress);
    ZKJobMasterFinder finder = new ZKJobMasterFinder(cnfg, jobID);
    finder.initialize();
    String jobMasterIPandPort = finder.getJobMasterIPandPort();
    if (jobMasterIPandPort == null) {
        LOG.info("Job Master has not joined yet. Will wait and try to get the address ...");
        jobMasterIPandPort = finder.waitAndGetJobMasterIPandPort(20000);
        LOG.info("Job Master address: " + jobMasterIPandPort);
    } else {
        LOG.info("Job Master address: " + jobMasterIPandPort);
    }
    finder.close();
    LOG.info("Done, exiting ...");
}

Example 99 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class SourceTaskDataLoaderMain method main.

public static void main(String[] args) throws InvalidArguments {
    if (args.length == 3) {
        workers = Integer.parseInt(args[0]);
        parallelism = Integer.parseInt(args[1]);
        dataSource = args[2];
        Config config = ResourceAllocator.loadConfig(new HashMap<>());
        JobConfig jobConfig = new JobConfig();
        jobConfig.put(WorkerConstants.WORKERS, workers);
        jobConfig.put(WorkerConstants.PARALLELISM, parallelism);
        jobConfig.put(MLDataObjectConstants.TRAINING_DATA_DIR, dataSource);
        Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
        jobBuilder.setJobName("SourceTaskDataLoader");
        jobBuilder.setWorkerClass(SourceTaskDataLoader.class.getName());
        jobBuilder.addComputeResource(2, 512, 1.0, workers);
        jobBuilder.setConfig(jobConfig);
        // now submit the job
        Twister2Submitter.submitJob(jobBuilder.build(), config);
    } else {
        throw new InvalidArguments("Invalid Arguments : <workers> <parallelism> <datafile_path>");
    }
}

Example 100 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskWorkerDataLoaderMain method main.

public static void main(String[] args) throws InvalidArguments {
    if (args.length == 3) {
        workers = Integer.parseInt(args[0]);
        parallelism = Integer.parseInt(args[1]);
        dataSource = args[2];
        Config config = ResourceAllocator.loadConfig(new HashMap<>());
        JobConfig jobConfig = new JobConfig();
        jobConfig.put(WorkerConstants.WORKERS, workers);
        jobConfig.put(WorkerConstants.PARALLELISM, parallelism);
        jobConfig.put(MLDataObjectConstants.TRAINING_DATA_DIR, dataSource);
        Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
        jobBuilder.setJobName("TaskWorkerDataLoader");
        jobBuilder.setWorkerClass(TaskWorkerDataLoader.class.getName());
        jobBuilder.addComputeResource(2, 512, 1.0, workers);
        jobBuilder.setConfig(jobConfig);
        // now submit the job
        Twister2Submitter.submitJob(jobBuilder.build(), config);
    } else {
        throw new InvalidArguments("Invalid Arguments : <workers> <parallelism> <datafile_path>");
    }
}