Examples with RequestedResources edu.iu.dsc.tws.rsched.spi.resource.RequestedResources used on opensource projects

Example 1 with RequestedResources

use of edu.iu.dsc.tws.rsched.spi.resource.RequestedResources in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method submitJob.

/**
 * Submit the job to the cluster
 *
 * @param job the actual job description
 */
public void submitJob(JobAPI.Job job, Config config) {
    // lets prepare the job files
    // String jobDirectory = prepareJobFilesOld(config, job);
    String jobDirectory = prepareJobFiles(config, job);
    String statemgrClass = SchedulerContext.stateManagerClass(config);
    if (statemgrClass == null) {
        throw new RuntimeException("The state manager class must be spcified");
    }
    String launcherClass = SchedulerContext.launcherClass(config);
    if (launcherClass == null) {
        throw new RuntimeException("The launcher class must be specified");
    }
    String uploaderClass = SchedulerContext.uploaderClass(config);
    if (uploaderClass == null) {
        throw new RuntimeException("The uploader class must be specified");
    }
    ILauncher launcher;
    IUploader uploader;
    IStateManager statemgr;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new JobSubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    LOG.log(Level.INFO, "Initialize state manager");
    // initialize the state manager
    statemgr.initialize(config);
    LOG.log(Level.INFO, "Initialize uploader");
    // now upload the content of the package
    uploader.initialize(config);
    // gives the url of the file to be uploaded
    LOG.log(Level.INFO, "Calling uploader to upload the package content");
    URI packageURI = uploader.uploadPackage(jobDirectory);
    // add scp address as a prefix to returned URI: user@ip
    String scpServerAdress = ScpContext.scpConnection(updatedConfig);
    String scpPath = scpServerAdress + ":" + packageURI.toString() + "/";
    LOG.log(Level.INFO, "SCP PATH to copy files from: " + scpPath);
    // this is a temporary solution
    // String packagesPath = "root@149.165.150.81:/root/.twister2/repository/";
    // String packagesPath = "149.165.150.81:~/.twister2/repository/";
    // now launch the launcher
    // Update the runtime config with the packageURI
    updatedConfig = Config.newBuilder().putAll(updatedConfig).put(SchedulerContext.TWISTER2_PACKAGES_PATH, scpPath).put(SchedulerContext.JOB_PACKAGE_URI, packageURI).build();
    // this is a handler chain based execution in resource allocator. We need to
    // make it more formal as such
    launcher.initialize(updatedConfig);
    RequestedResources requestedResources = buildRequestedResources(updatedJob);
    if (requestedResources == null) {
        throw new RuntimeException("Failed to build the requested resources");
    }
    launcher.launch(requestedResources, updatedJob);
}

Example 2 with RequestedResources

use of edu.iu.dsc.tws.rsched.spi.resource.RequestedResources in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method buildRequestedResources.

private RequestedResources buildRequestedResources(JobAPI.Job job) {
    JobAPI.JobResources jobResources = job.getJobResources();
    int noOfContainers = jobResources.getNoOfContainers();
    ResourceContainer container = new ResourceContainer((int) jobResources.getContainer().getAvailableCPU(), (int) jobResources.getContainer().getAvailableMemory(), (int) jobResources.getContainer().getAvailableDisk());
    return new RequestedResources(noOfContainers, container);
}