Examples with LocalFileSystemUploader edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader used on opensource projects

Example 1 with LocalFileSystemUploader

use of edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method submitJob.

/**
 * Submit the job to the cluster
 */
public Twister2JobState submitJob() {
    // check whether uploader and launcher classes are specified
    checkUploaderAndLauncherClasses();
    String jobDirectory = prepareJobFiles();
    // upload the job package
    IUploader uploader = uploadJobPackage();
    // initialize the launcher and launch the job
    ILauncher launcher = initializeLauncher();
    Twister2JobState launchState = launcher.launch(job);
    // clear job files and return
    if (!launchState.isRequestGranted()) {
        launcher.close();
        if (!SchedulerContext.isLocalFileSystemUploader(config)) {
            uploader.undo();
        }
        // clear temporary twister2 files
        clearTemporaryFiles(jobDirectory);
        return launchState;
    }
    // clear job resources and return
    if (!uploader.complete()) {
        LOG.log(Level.SEVERE, "Transferring the job package failed." + "\n++++++++++++++++++ Aborting submission ++++++++++++++++++");
        launcher.killJob(job.getJobId());
        launchState.setRequestGranted(false);
        launcher.close();
        // clear temporary twister2 files
        clearTemporaryFiles(jobDirectory);
        return launchState;
    }
    // job is submitted successfully
    // close the launcher
    launcher.close();
    // copy the job package to the local repository
    if (CheckpointingContext.isCheckpointingEnabled(config) && !SchedulerContext.isLocalFileSystemUploader(config)) {
        IUploader localUploader = new LocalFileSystemUploader();
        localUploader.initialize(config, job.getJobId());
        URI savedPackage = localUploader.uploadPackage(jobDirectory);
        LOG.info("Saved Job Package to Directory: " + savedPackage.getPath());
    }
    if (!CheckpointingContext.isCheckpointingEnabled(config) && SchedulerContext.clusterType(config).equals("standalone") && SchedulerContext.isLocalFileSystemUploader(config)) {
        uploader.undo();
    }
    // clear temporary twister2 files
    clearTemporaryFiles(jobDirectory);
    return launchState;
}

Example 2 with LocalFileSystemUploader

use of edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method killJob.

/**
 * Kill the job
 *
 * @param jobID of the job to kill
 */
public static void killJob(String jobID, Config cnfg) {
    String launcherClass = SchedulerContext.launcherClass(cnfg);
    if (launcherClass == null) {
        throw new RuntimeException("The launcher class must be specified");
    }
    ILauncher launcher;
    ClassLoader classLoader = ResourceAllocator.class.getClassLoader();
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(classLoader, launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // initialize the launcher and terminate the job
    launcher.initialize(cnfg);
    boolean killed = launcher.killJob(jobID);
    if (!killed) {
        LOG.log(Level.SEVERE, "Could not kill the job");
    }
    String uploaderClass = SchedulerContext.uploaderClass(cnfg);
    if (uploaderClass == null) {
        throw new RuntimeException("The uploader class must be specified");
    }
    IUploader uploader;
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(classLoader, uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    uploader.initialize(cnfg, jobID);
    uploader.undo();
    launcher.close();
    uploader.close();
    if (KubernetesContext.isKubernetesCluster(cnfg)) {
        KubernetesController.close();
    }
    if (CheckpointingContext.isCheckpointingEnabled(cnfg) && !SchedulerContext.uploaderClass(cnfg).equals("edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader")) {
        IUploader localUploader = new LocalFileSystemUploader();
        localUploader.initialize(cnfg, jobID);
        localUploader.undo();
    }
}