Examples with JobConfig edu.iu.dsc.tws.api.JobConfig used on opensource projects

Example 31 with JobConfig

use of edu.iu.dsc.tws.api.JobConfig in project twister2 by DSC-SPIDAL.

the class BasicTaskSchedulingJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    // build the job
    BasicJob basicJob = BasicJob.newBuilder().setName("basic-testing").setContainerClass(TaskScheduling.class.getName()).setRequestResource(new ResourceContainer(2, 1024), 2).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitContainerJob(basicJob, config);
}

Example 32 with JobConfig

use of edu.iu.dsc.tws.api.JobConfig in project beam by apache.

the class Twister2Runner method run.

@Override
public PipelineResult run(Pipeline pipeline) {
    // create a worker and pass in the pipeline and then do the translation
    Twister2PipelineExecutionEnvironment env = new Twister2PipelineExecutionEnvironment(options);
    LOG.info("Translating pipeline to Twister2 program.");
    pipeline.replaceAll(getDefaultOverrides());
    // TODO(BEAM-10670): Use SDF read as default when we address performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    env.translate(pipeline);
    setupSystem(options);
    Map configMap = new HashMap();
    JobConfig jobConfig = new JobConfig();
    if (isLocalMode(options)) {
        options.setParallelism(1);
        configMap.put(SIDEINPUTS, extractNames(env.getSideInputs()));
        configMap.put(LEAVES, extractNames(env.getLeaves()));
        configMap.put(GRAPH, env.getTSetGraph());
        configMap.put("twister2.network.buffer.size", 32000);
        configMap.put("twister2.network.sendBuffer.count", options.getParallelism());
        LOG.warning("Twister2 Local Mode currently only supports single worker");
    } else {
        jobConfig.put(SIDEINPUTS, extractNames(env.getSideInputs()));
        jobConfig.put(LEAVES, extractNames(env.getLeaves()));
        jobConfig.put(GRAPH, env.getTSetGraph());
    }
    Config config = ResourceAllocator.loadConfig(configMap);
    int workers = options.getParallelism();
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName(options.getJobName()).setWorkerClass(BeamBatchWorker.class).addComputeResource(options.getWorkerCPUs(), options.getRamMegaBytes(), workers).setConfig(jobConfig).build();
    Twister2JobState jobState;
    if (isLocalMode(options)) {
        jobState = LocalSubmitter.submitJob(twister2Job, config);
    } else {
        jobState = Twister2Submitter.submitJob(twister2Job, config);
    }
    Twister2PipelineResult result = new Twister2PipelineResult(jobState);
    return result;
}

Example 33 with JobConfig

use of edu.iu.dsc.tws.api.JobConfig in project beam by apache.

the class Twister2Runner method runTest.

public PipelineResult runTest(Pipeline pipeline) {
    // create a worker and pass in the pipeline and then do the translation
    Twister2PipelineExecutionEnvironment env = new Twister2PipelineExecutionEnvironment(options);
    LOG.info("Translating pipeline to Twister2 program.");
    pipeline.replaceAll(getDefaultOverrides());
    // TODO(BEAM-10670): Use SDF read as default when we address performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    env.translate(pipeline);
    setupSystemTest(options);
    Map configMap = new HashMap();
    configMap.put(SIDEINPUTS, extractNames(env.getSideInputs()));
    configMap.put(LEAVES, extractNames(env.getLeaves()));
    configMap.put(GRAPH, env.getTSetGraph());
    configMap.put("twister2.network.buffer.size", 32000);
    configMap.put("twister2.network.sendBuffer.count", options.getParallelism());
    Config config = ResourceAllocator.loadConfig(configMap);
    JobConfig jobConfig = new JobConfig();
    int workers = options.getParallelism();
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName(options.getJobName()).setWorkerClass(BeamBatchWorker.class).addComputeResource(options.getWorkerCPUs(), options.getRamMegaBytes(), workers).setConfig(jobConfig).build();
    Twister2JobState jobState = LocalSubmitter.submitJob(twister2Job, config);
    Twister2PipelineResult result = new Twister2PipelineResult(jobState);
    // TODO: Need to fix the check for "RUNNING" once fix for this is done on Twister2 end.
    if (result.state == PipelineResult.State.FAILED) {
        throw new RuntimeException("Pipeline execution failed", jobState.getCause());
    }
    return result;
}