Examples with IUploader edu.iu.dsc.tws.api.scheduler.IUploader used on opensource projects

Example 1 with IUploader

use of edu.iu.dsc.tws.api.scheduler.IUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method submitJob.

/**
 * Submit the job to the cluster
 */
public Twister2JobState submitJob() {
    // check whether uploader and launcher classes are specified
    checkUploaderAndLauncherClasses();
    String jobDirectory = prepareJobFiles();
    // upload the job package
    IUploader uploader = uploadJobPackage();
    // initialize the launcher and launch the job
    ILauncher launcher = initializeLauncher();
    Twister2JobState launchState = launcher.launch(job);
    // clear job files and return
    if (!launchState.isRequestGranted()) {
        launcher.close();
        if (!SchedulerContext.isLocalFileSystemUploader(config)) {
            uploader.undo();
        }
        // clear temporary twister2 files
        clearTemporaryFiles(jobDirectory);
        return launchState;
    }
    // clear job resources and return
    if (!uploader.complete()) {
        LOG.log(Level.SEVERE, "Transferring the job package failed." + "\n++++++++++++++++++ Aborting submission ++++++++++++++++++");
        launcher.killJob(job.getJobId());
        launchState.setRequestGranted(false);
        launcher.close();
        // clear temporary twister2 files
        clearTemporaryFiles(jobDirectory);
        return launchState;
    }
    // job is submitted successfully
    // close the launcher
    launcher.close();
    // copy the job package to the local repository
    if (CheckpointingContext.isCheckpointingEnabled(config) && !SchedulerContext.isLocalFileSystemUploader(config)) {
        IUploader localUploader = new LocalFileSystemUploader();
        localUploader.initialize(config, job.getJobId());
        URI savedPackage = localUploader.uploadPackage(jobDirectory);
        LOG.info("Saved Job Package to Directory: " + savedPackage.getPath());
    }
    if (!CheckpointingContext.isCheckpointingEnabled(config) && SchedulerContext.clusterType(config).equals("standalone") && SchedulerContext.isLocalFileSystemUploader(config)) {
        uploader.undo();
    }
    // clear temporary twister2 files
    clearTemporaryFiles(jobDirectory);
    return launchState;
}

Example 2 with IUploader

use of edu.iu.dsc.tws.api.scheduler.IUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method killJob.

/**
 * Kill the job
 *
 * @param jobID of the job to kill
 */
public static void killJob(String jobID, Config cnfg) {
    String launcherClass = SchedulerContext.launcherClass(cnfg);
    if (launcherClass == null) {
        throw new RuntimeException("The launcher class must be specified");
    }
    ILauncher launcher;
    ClassLoader classLoader = ResourceAllocator.class.getClassLoader();
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(classLoader, launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // initialize the launcher and terminate the job
    launcher.initialize(cnfg);
    boolean killed = launcher.killJob(jobID);
    if (!killed) {
        LOG.log(Level.SEVERE, "Could not kill the job");
    }
    String uploaderClass = SchedulerContext.uploaderClass(cnfg);
    if (uploaderClass == null) {
        throw new RuntimeException("The uploader class must be specified");
    }
    IUploader uploader;
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(classLoader, uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    uploader.initialize(cnfg, jobID);
    uploader.undo();
    launcher.close();
    uploader.close();
    if (KubernetesContext.isKubernetesCluster(cnfg)) {
        KubernetesController.close();
    }
    if (CheckpointingContext.isCheckpointingEnabled(cnfg) && !SchedulerContext.uploaderClass(cnfg).equals("edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader")) {
        IUploader localUploader = new LocalFileSystemUploader();
        localUploader.initialize(cnfg, jobID);
        localUploader.undo();
    }
}

Example 3 with IUploader

use of edu.iu.dsc.tws.api.scheduler.IUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method resubmitJob.

/**
 * Resubmit the job to the cluster
 */
public Twister2JobState resubmitJob() {
    // check whether uploader and launcher classes are specified
    checkUploaderAndLauncherClasses();
    // upload the job package if it is not local upoader
    IUploader uploader = null;
    if (!SchedulerContext.isLocalFileSystemUploader(config)) {
        uploader = uploadJobPackage();
    }
    // initialize the launcher and launch the job
    ILauncher launcher = initializeLauncher();
    Twister2JobState launchState = launcher.launch(job);
    // clear job files and return
    if (!launchState.isRequestGranted()) {
        launcher.close();
        if (uploader != null) {
            uploader.undo();
        }
        return launchState;
    }
    // clear job resources and return
    if (uploader != null && !uploader.complete()) {
        LOG.log(Level.SEVERE, "Transferring the job package failed." + "\n++++++++++++++++++ Aborting submission ++++++++++++++++++");
        launcher.killJob(job.getJobId());
        launchState.setRequestGranted(false);
        launcher.close();
        return launchState;
    }
    // job is submitted successfully
    // close the launcher
    launcher.close();
    return launchState;
}

Example 4 with IUploader

use of edu.iu.dsc.tws.api.scheduler.IUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method uploadJobPackage.

private IUploader uploadJobPackage() {
    String uploaderClass = SchedulerContext.uploaderClass(config);
    IUploader uploader;
    ClassLoader classLoader = ResourceAllocator.class.getClassLoader();
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(classLoader, uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    LOG.fine("Initialize uploader");
    // now upload the content of the package
    uploader.initialize(config, job.getJobId());
    // gives the url of the file to be uploaded
    LOG.fine("Calling uploader to upload the job package");
    long start = System.currentTimeMillis();
    String jobDirectory = SchedulerContext.temporaryPackagesPath(config);
    URI packageURI = uploader.uploadPackage(jobDirectory);
    long delay = System.currentTimeMillis() - start;
    LOG.info("Job package upload started. It took: " + delay + "ms");
    // add scp address as a prefix to returned URI: user@ip
    String scpServerAdress = ScpContext.scpConnection(config);
    String scpPath = scpServerAdress;
    if (packageURI != null) {
        scpPath += ":" + packageURI.toString() + "/";
    }
    LOG.fine("SCP PATH to copy files from: " + scpPath);
    // now launch the launcher
    // Update the runtime config with the packageURI
    config = Config.newBuilder().putAll(config).put(SchedulerContext.TWISTER2_PACKAGES_PATH, scpPath).put(SchedulerContext.JOB_PACKAGE_URI, packageURI).build();
    return uploader;
}