Examples with UploaderException edu.iu.dsc.tws.api.scheduler.UploaderException used on opensource projects

Example 6 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class S3Uploader method uploadPackage.

@Override
public URI uploadPackage(String sourceLocation) throws UploaderException {
    localJobPackFile = sourceLocation + "/" + SchedulerContext.jobPackageFileName(config);
    s3File = S3Context.s3BucketName(config) + "/" + JobUtils.createJobPackageFileName(jobID);
    long linkExpDur = S3Context.linkExpirationDuration(config);
    String cmd = String.format("aws s3 presign %s --expires-in %s", s3File, linkExpDur);
    LOG.fine("cmd for s3 URL Generation: " + cmd);
    String[] fullCmd = { "bash", "-c", cmd };
    String url;
    Process p = null;
    try {
        p = Runtime.getRuntime().exec(fullCmd);
        BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
        url = reader.readLine();
        p.waitFor();
    } catch (IOException e) {
        throw new UploaderException("Exception when executing the command: " + cmd, e);
    } catch (InterruptedException e) {
        throw new UploaderException("Exception when waiting the script to complete: " + cmd, e);
    }
    int exitCode = p.exitValue();
    if (exitCode == 0) {
        LOG.fine("Job Package Download URL: " + url);
        try {
            URI uri = new URI(url);
            // start uploader thread
            start();
            // if this is a resubmitted job, wait for the transfer to complete
            if (CheckpointingContext.startingFromACheckpoint(config)) {
                waitComplete();
            }
            return uri;
        } catch (URISyntaxException e) {
            throw new UploaderException("Can not generate URI for download link: " + url, e);
        }
    } else {
        String failMsg = String.format("Some error occurred when presigning job package %s at s3: %s", localJobPackFile, s3File);
        throw new UploaderException(failMsg);
    }
}

Example 7 with UploaderException

use of edu.iu.dsc.tws.api.scheduler.UploaderException in project twister2 by DSC-SPIDAL.

the class UploaderToWebServers method uploadPackage.

@Override
public URI uploadPackage(String sourceLocation) throws UploaderException {
    localJobPackageFile = sourceLocation + File.separator + SchedulerContext.jobPackageFileName(config);
    // start uploader thread
    start();
    // partial files may be served to workers
    if (CheckpointingContext.startingFromACheckpoint(config)) {
        // wait for the uploaders to start
        while (webServerPodNames.size() != uploadersToWebServers.size()) {
            try {
                Thread.sleep(50);
            } catch (InterruptedException e) {
            // ignore the exception
            }
        }
        LOG.fine("Wait uploading to web servers to finish ...");
        complete();
        LOG.info("Uploading to web servers finished ...");
    }
    String uri = KubernetesContext.uploaderWebServer(config) + "/" + JobUtils.createJobPackageFileName(jobID);
    try {
        return new URI(uri);
    } catch (URISyntaxException e) {
        LOG.log(Level.SEVERE, "Can not generate URI for uploader web server: " + uri, e);
        throw new UploaderException("Can not generate URI for download link: " + uri, e);
    }
}