Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 26 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class IntervalWindowTopology method main.

public static void main(String[] args) {
    Config config = ResourceAllocator.loadConfig(Collections.emptyMap());
    JobConfig jobConfig = new JobConfig();
    jobConfig.put("topology.message.timeout.secs", 100000);
    Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
    jobBuilder.setJobName("sliding-window-example");
    jobBuilder.setWorkerClass(IntervalWindowTopology.class.getName());
    jobBuilder.setConfig(jobConfig);
    jobBuilder.addComputeResource(1, 512, 1);
    // now submit the job
    Twister2Submitter.submitJob(jobBuilder.build(), config);
}

Example 27 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TumblingWindowTopology method main.

public static void main(String[] args) {
    Config config = ResourceAllocator.loadConfig(Collections.emptyMap());
    JobConfig jobConfig = new JobConfig();
    Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
    jobBuilder.setJobName("tumbling-window-example");
    jobBuilder.setWorkerClass(TumblingWindowTopology.class.getName());
    jobBuilder.setConfig(jobConfig);
    jobBuilder.addComputeResource(1, 512, 1);
    // now submit the job
    Twister2Submitter.submitJob(jobBuilder.build(), config);
}

Example 28 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.FINE, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    // Independent Graph and it has collector
    ComputeGraph firstGraph = buildFirstGraph(2, config);
    // Dependent Graph and it has collector
    ComputeGraph secondGraph = buildSecondGraph(4, config);
    // Dependent Graph and it has receptor to receive the input from second graph or first graph
    ComputeGraph thirdGraph = buildThirdGraph(4, config);
    ComputeGraph[] computeGraphs = new ComputeGraph[] { firstGraph, secondGraph, thirdGraph };
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    // Get the execution plan for the third task graph
    ExecutionPlan thirdGraphExecutionPlan = taskExecutor.plan(thirdGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    taskExecutor.execute(thirdGraph, thirdGraphExecutionPlan);
    // This is to test all the three graphs as dependent
    /*Map<String, ExecutionPlan> taskExecutionPlan = taskExecutor.plan(computeGraphs);
    for (Map.Entry<String, ExecutionPlan> planEntry : taskExecutionPlan.entrySet()) {
      String graphName = planEntry.getKey();
      if (graphName.equals(computeGraphs[0].getGraphName())) {
        taskExecutor.execute(computeGraphs[0], planEntry.getValue());
      } else if (graphName.equals(computeGraphs[1].getGraphName())) {
        taskExecutor.execute(computeGraphs[1], planEntry.getValue());
      } else {
        taskExecutor.execute(computeGraphs[2], planEntry.getValue());
      }
    }*/
    cEnv.close();
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}

Example 29 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method main.

public static void main(String[] args) throws ParseException {
    LOG.log(Level.INFO, "Batch Task Graph Example");
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    Options options = new Options();
    options.addOption(DataObjectConstants.WORKERS, true, "Workers");
    options.addOption(DataObjectConstants.PARALLELISM_VALUE, true, "parallelism");
    options.addOption(DataObjectConstants.DSIZE, true, "dsize");
    @SuppressWarnings("deprecation") CommandLineParser commandLineParser = new DefaultParser();
    CommandLine cmd = commandLineParser.parse(options, args);
    int workers = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.WORKERS));
    int parallelismValue = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.PARALLELISM_VALUE));
    int dsize = Integer.parseInt(cmd.getOptionValue(DataObjectConstants.DSIZE));
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    jobConfig.put(DataObjectConstants.WORKERS, Integer.toString(workers));
    jobConfig.put(DataObjectConstants.PARALLELISM_VALUE, Integer.toString(parallelismValue));
    jobConfig.put(DataObjectConstants.DSIZE, Integer.toString(dsize));
    Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
    jobBuilder.setJobName("BatchScheduler-test");
    jobBuilder.setWorkerClass(BatchTaskSchedulerExample.class.getName());
    jobBuilder.addComputeResource(2, 2048, 1.0, workers);
    jobBuilder.setConfig(jobConfig);
    // now submit the job
    Twister2Submitter.submitJob(jobBuilder.build(), config);
}

Example 30 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class JobMasterClientExample method main.

/**
 * a test class to run JMWorkerAgent
 * First, a JobMaster instance should be started on a machine
 * This client should connect to that server
 * <p>
 * It reads config files from conf/kubernetes directory
 * It uses the first ComputeResource in that config file as the ComputeResource of this worker
 * Number of workers is the number of workers in the first ComputeResource
 * <p>
 * When all workers joined, they get the full worker list
 * Then, each worker sends a barrier message
 * Then, each worker sends a completed message and closes
 */
public static void main(String[] args) {
    if (args.length != 3) {
        LOG.severe("Provide jmAddress workerID and jobID as parameters.");
        return;
    }
    String jmAddress = args[0];
    int workerID = Integer.parseInt(args[1]);
    String jobID = args[2];
    // we assume that the twister2Home is the current directory
    // String configDir = "../twister2/config/src/yaml/";
    String configDir = "";
    String twister2Home = Paths.get(configDir).toAbsolutePath().toString();
    Config config1 = ConfigLoader.loadConfig(twister2Home, "conf/kubernetes");
    Config config2 = ConfigLoader.loadConfig(twister2Home, "conf/common");
    Config config = updateConfig(config1, config2, jmAddress);
    LOG.info("Loaded: " + config.size() + " configuration parameters.");
    Twister2Job twister2Job = Twister2Job.loadTwister2Job(config, null);
    twister2Job.setJobID(jobID);
    JobAPI.Job job = twister2Job.serialize();
    LOG.info("workerID: " + workerID);
    LOG.info("jobID: " + jobID);
    simulateClient(config, job, workerID);
}