Examples with UploaderException edu.iu.dsc.tws.api.scheduler.UploaderException used on opensource projects

Example 1 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class HdfsUploader method uploadPackage.

@Override
public URI uploadPackage(String sourceLocation) throws UploaderException {
    // first, check if the topology package exists
    File file = new File(sourceLocation);
    String fileName = file.getName();
    packageURI = TypeUtils.getURI(destTopologyDirectoryURI + "/" + fileName);
    if (!isLocalFileExists(sourceLocation)) {
        throw new UploaderException(String.format("Expected topology package file to be uploaded does not exist at '%s'", sourceLocation));
    }
    // if the dest directory does not exist, create it.
    if (!controller.exists(destTopologyDirectoryURI)) {
        LOG.info(String.format("The destination directory does not exist. Creating it now at URI '%s'", destTopologyDirectoryURI));
        if (!controller.mkdirs(destTopologyDirectoryURI)) {
            throw new UploaderException(String.format("Failed to create directory for topology package at URI '%s'", destTopologyDirectoryURI));
        }
    } else {
        // if the destination file exists, write a log message
        LOG.info(String.format("Target topology file already exists at '%s'. Overwriting it now", packageURI.toString()));
    }
    // copy the topology package to target working directory
    LOG.info(String.format("Uploading topology package at '%s' to target HDFS at '%s'", sourceLocation, packageURI.toString()));
    if (!controller.copyFromLocalFile(sourceLocation, packageURI.toString())) {
        throw new UploaderException(String.format("Failed to upload the topology package at '%s' to: '%s'", sourceLocation, packageURI.toString()));
    }
    try {
        return new URI(destTopologyDirectoryURI + '/' + fileName);
    } catch (URISyntaxException e) {
        throw new RuntimeException("Invalid file path for topology package destination: " + destTopologyDirectoryURI, e);
    }
}

Example 2 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class LocalFileSystemUploader method uploadPackage.

@Override
public URI uploadPackage(String sourceLocation) throws UploaderException {
    // we shouldn't come here naturally as a jar file is needed for us to get here
    File file = new File(sourceLocation);
    boolean fileExists = file.isDirectory();
    if (!fileExists) {
        throw new UploaderException(String.format("Job package does not exist at '%s'", sourceLocation));
    }
    String directoryName = file.getName();
    // get the directory containing the file
    Path filePath = Paths.get(destinationDirectory);
    File parentDirectory = filePath.toFile();
    assert parentDirectory != null;
    // if the dest directory does not exist, create it.
    if (!parentDirectory.exists()) {
        LOG.log(Level.FINE, String.format("Target directory does not exist. Creating it now at %s", parentDirectory.getPath()));
        if (!parentDirectory.mkdirs()) {
            throw new UploaderException(String.format("Failed to create directory for topology package at %s", parentDirectory.getPath()));
        }
    }
    // if the dest file exists, write a log message
    fileExists = new File(filePath.toString()).isFile();
    if (fileExists) {
        LOG.fine(String.format("Target job package already exists at '%s'. Overwriting it now", filePath.toString()));
    }
    // copy the job package to target working directory
    LOG.log(Level.FINE, String.format("Copying job directory at '%s' to target directory '%s'", sourceLocation, filePath.toString()));
    try {
        FileUtils.copyDirectory(sourceLocation, destinationDirectory);
        return new URI(destinationDirectory);
    } catch (URISyntaxException e) {
        throw new RuntimeException("Invalid file path for topology package destination: " + destinationDirectory, e);
    } catch (IOException e) {
        throw new RuntimeException(String.format("Failed to copy directory %s to %s", sourceLocation, destinationDirectory));
    }
}

Example 3 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class ScpUploader method uploadPackage.

@Override
public URI uploadPackage(String sourceLocation) throws UploaderException {
    String source = sourceLocation + "/";
    File file = new File(source);
    String fileName = file.getName();
    boolean dirExist = file.isDirectory();
    if (!dirExist) {
        throw new UploaderException(String.format("Job package does not exist at '%s'", source));
    }
    LOG.log(Level.INFO, String.format("Uploading the file from local" + " file system to remote machine: %s -> %s.", source, destinationDirectory));
    try {
        if (!this.controller.copyFromLocalDirectory(source, destinationDirectory)) {
            throw new UploaderException(String.format("Failed to upload the file from local file system to remote machine: %s -> %s.", source, destinationDirectory));
        }
        LOG.log(Level.INFO, String.format("Uploaded to remote machine: %s -> %s.", source, destinationDirectory));
        return new URI(destinationDirectory);
    } catch (URISyntaxException e) {
        throw new RuntimeException("Invalid file path for topology package destination: " + destinationDirectory, e);
    }
}

Example 4 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method killJob.

/**
 * Kill the job
 *
 * @param jobID of the job to kill
 */
public static void killJob(String jobID, Config cnfg) {
    String launcherClass = SchedulerContext.launcherClass(cnfg);
    if (launcherClass == null) {
        throw new RuntimeException("The launcher class must be specified");
    }
    ILauncher launcher;
    ClassLoader classLoader = ResourceAllocator.class.getClassLoader();
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(classLoader, launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // initialize the launcher and terminate the job
    launcher.initialize(cnfg);
    boolean killed = launcher.killJob(jobID);
    if (!killed) {
        LOG.log(Level.SEVERE, "Could not kill the job");
    }
    String uploaderClass = SchedulerContext.uploaderClass(cnfg);
    if (uploaderClass == null) {
        throw new RuntimeException("The uploader class must be specified");
    }
    IUploader uploader;
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(classLoader, uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    uploader.initialize(cnfg, jobID);
    uploader.undo();
    launcher.close();
    uploader.close();
    if (KubernetesContext.isKubernetesCluster(cnfg)) {
        KubernetesController.close();
    }
    if (CheckpointingContext.isCheckpointingEnabled(cnfg) && !SchedulerContext.uploaderClass(cnfg).equals("edu.iu.dsc.tws.rsched.uploaders.localfs.LocalFileSystemUploader")) {
        IUploader localUploader = new LocalFileSystemUploader();
        localUploader.initialize(cnfg, jobID);
        localUploader.undo();
    }
}

Example 5 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method uploadJobPackage.

private IUploader uploadJobPackage() {
    String uploaderClass = SchedulerContext.uploaderClass(config);
    IUploader uploader;
    ClassLoader classLoader = ResourceAllocator.class.getClassLoader();
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(classLoader, uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    LOG.fine("Initialize uploader");
    // now upload the content of the package
    uploader.initialize(config, job.getJobId());
    // gives the url of the file to be uploaded
    LOG.fine("Calling uploader to upload the job package");
    long start = System.currentTimeMillis();
    String jobDirectory = SchedulerContext.temporaryPackagesPath(config);
    URI packageURI = uploader.uploadPackage(jobDirectory);
    long delay = System.currentTimeMillis() - start;
    LOG.info("Job package upload started. It took: " + delay + "ms");
    // add scp address as a prefix to returned URI: user@ip
    String scpServerAdress = ScpContext.scpConnection(config);
    String scpPath = scpServerAdress;
    if (packageURI != null) {
        scpPath += ":" + packageURI.toString() + "/";
    }
    LOG.fine("SCP PATH to copy files from: " + scpPath);
    // now launch the launcher
    // Update the runtime config with the packageURI
    config = Config.newBuilder().putAll(config).put(SchedulerContext.TWISTER2_PACKAGES_PATH, scpPath).put(SchedulerContext.JOB_PACKAGE_URI, packageURI).build();
    return uploader;
}