Examples with Sandbox org.gridlab.gat.resources.cpi.Sandbox used on opensource projects

Example 1 with Sandbox

use of org.gridlab.gat.resources.cpi.Sandbox in project compss by bsc-wdc.

the class LsfResourceBrokerAdaptor method submitJob.

/*
     * (non-Javadoc)
     * 
     * @see org.gridlab.gat.resources.ResourceBroker#submitJob(org.gridlab.gat.resources.JobDescription)
     */
public Job submitJob(AbstractJobDescription abstractDescription, MetricListener listener, String metricDefinitionName) throws GATInvocationException {
    if (!(abstractDescription instanceof JobDescription)) {
        throw new GATInvocationException("can only handle JobDescriptions: " + abstractDescription.getClass());
    }
    JobDescription description = (JobDescription) abstractDescription;
    SoftwareDescription sd = description.getSoftwareDescription();
    if (sd == null) {
        throw new GATInvocationException("The job description does not contain a software description");
    }
    if (description.getProcessCount() < 1) {
        throw new GATInvocationException("Adaptor cannot handle: process count < 1: " + description.getProcessCount());
    }
    if (description.getResourceCount() != 1) {
        throw new GATInvocationException("Adaptor cannot handle: resource count > 1: " + description.getResourceCount());
    }
    String home = System.getProperty("user.home");
    if (home == null) {
        throw new GATInvocationException("lsf broker could not get user home dir");
    }
    Sandbox sandbox = new Sandbox(gatContext, description, "localhost", home, true, true, false, false);
    LsfJob lsfJob = new LsfJob(gatContext, description, sandbox);
    Job job = null;
    if (description instanceof WrapperJobDescription) {
        WrapperJobCpi tmp = new WrapperJobCpi(gatContext, lsfJob, listener, metricDefinitionName);
        listener = tmp;
        job = tmp;
    } else {
        job = lsfJob;
    }
    if (listener != null && metricDefinitionName != null) {
        Metric metric = lsfJob.getMetricDefinitionByName(metricDefinitionName).createMetric(null);
        lsfJob.addMetricListener(listener, metric);
    }
    lsfJob.setState(Job.JobState.PRE_STAGING);
    lsfJob.waitForTrigger(Job.JobState.PRE_STAGING);
    sandbox.prestage();
    String exe;
    if (sandbox.getResolvedExecutable() != null) {
        exe = sandbox.getResolvedExecutable().getPath();
    // try to set the executable bit, it might be lost
    /* CDIAZ: The command "exe" can be also in a remote host
             * 		  The command must have the right permissions in the remote host
            try {
                new CommandRunner("chmod", "+x", exe);
            } catch (Throwable t) {
                // ignore
            }
            */
    } else {
        exe = getExecutable(description);
    }
    String[] args = getArgumentsArray(description);
    // Directory where the lsf command will be executed
    java.io.File f = new java.io.File(sandbox.getSandboxPath());
    if (!f.exists()) {
        throw new GATInvocationException("Unable to find directory " + f.getAbsolutePath());
    }
    // Check and set the environment for a blaunch command
    Map<String, Object> env = sd.getEnvironment();
    this.prepareBLaunchEnv(env);
    // Encapsulate the original command into a blaunch command
    String host = brokerURI.getHost();
    String blExe = this.getBlaunchCommand();
    String[] blArgs = this.getBlaunchArgs(host, exe, args);
    ProcessBundle bundle = new ProcessBundle(description.getProcessCount(), blExe, blArgs, f, env);
    lsfJob.setSubmissionTime();
    lsfJob.setState(Job.JobState.SCHEDULED);
    try {
        lsfJob.setState(Job.JobState.RUNNING);
        lsfJob.waitForTrigger(Job.JobState.RUNNING);
        lsfJob.setStartTime();
        bundle.startBundle();
        lsfJob.setProcess(bundle);
    } catch (IOException e) {
        throw new CommandNotFoundException("LsfResourceBrokerAdaptor", e);
    }
    if (!sd.streamingStderrEnabled()) {
        try {
            if (sd.getStderr() != null) {
                OutputStream err = GAT.createFileOutputStream(gatContext, sd.getStderr());
                // to file
                StreamForwarder forwarder = new StreamForwarder(bundle.getStderr(), err, sd.getExecutable() + " [stderr]");
                lsfJob.setErrorStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stderr forwarder to file " + sd.getStderr());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStderr(), null, sd.getExecutable() + " [stderr]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stderr!", e);
        }
    }
    if (!sd.streamingStdoutEnabled()) {
        // read away the stdout
        try {
            if (sd.getStdout() != null) {
                // to file
                OutputStream out = GAT.createFileOutputStream(gatContext, sd.getStdout());
                StreamForwarder forwarder = new StreamForwarder(bundle.getStdout(), out, sd.getExecutable() + " [stdout]");
                lsfJob.setOutputStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stdout forwarder to file " + sd.getStdout());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStdout(), null, sd.getExecutable() + " [stdout]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stdout!", e);
        }
    }
    if (!sd.streamingStdinEnabled() && sd.getStdin() != null) {
        // forward the stdin from file
        try {
            InputStream in = GAT.createFileInputStream(gatContext, sd.getStdin());
            bundle.setStdin(sd.getExecutable(), in);
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file input stream for stdin!", e);
        }
    }
    lsfJob.monitorState();
    return job;
}

Example 2 with Sandbox

use of org.gridlab.gat.resources.cpi.Sandbox in project compss by bsc-wdc.

the class LoadLevelerResourceBrokerAdaptor method submitJob.

/*
     * (non-Javadoc)
     * 
     * @see org.gridlab.gat.resources.ResourceBroker#submitJob(org.gridlab.gat.resources.JobDescription)
     */
public Job submitJob(AbstractJobDescription abstractDescription, MetricListener listener, String metricDefinitionName) throws GATInvocationException {
    if (!(abstractDescription instanceof JobDescription)) {
        throw new GATInvocationException("can only handle JobDescriptions: " + abstractDescription.getClass());
    }
    JobDescription description = (JobDescription) abstractDescription;
    SoftwareDescription sd = description.getSoftwareDescription();
    if (sd == null) {
        throw new GATInvocationException("The job description does not contain a software description");
    }
    if (description.getProcessCount() < 1) {
        throw new GATInvocationException("Adaptor cannot handle: process count < 1: " + description.getProcessCount());
    }
    if (description.getResourceCount() != 1) {
        throw new GATInvocationException("Adaptor cannot handle: resource count > 1: " + description.getResourceCount());
    }
    String home = System.getProperty("user.home");
    if (home == null) {
        throw new GATInvocationException("loadleveler broker could not get user home dir");
    }
    Sandbox sandbox = new Sandbox(gatContext, description, "localhost", home, true, true, false, false);
    LoadLevelerJob loadlevelerJob = new LoadLevelerJob(gatContext, description, sandbox);
    Job job = null;
    if (description instanceof WrapperJobDescription) {
        WrapperJobCpi tmp = new WrapperJobCpi(gatContext, loadlevelerJob, listener, metricDefinitionName);
        listener = tmp;
        job = tmp;
    } else {
        job = loadlevelerJob;
    }
    if (listener != null && metricDefinitionName != null) {
        Metric metric = loadlevelerJob.getMetricDefinitionByName(metricDefinitionName).createMetric(null);
        loadlevelerJob.addMetricListener(listener, metric);
    }
    loadlevelerJob.setState(Job.JobState.PRE_STAGING);
    loadlevelerJob.waitForTrigger(Job.JobState.PRE_STAGING);
    sandbox.prestage();
    String exe;
    if (sandbox.getResolvedExecutable() != null) {
        exe = sandbox.getResolvedExecutable().getPath();
    // try to set the executable bit, it might be lost
    /* CDIAZ: The command "exe" can be also in a remote host
             * 		  The command must have the right permissions in the remote host
            try {
                new CommandRunner("chmod", "+x", exe);
            } catch (Throwable t) {
                // ignore
            }
            */
    } else {
        exe = getExecutable(description);
    }
    String[] args = getArgumentsArray(description);
    // Directory where the loadleveler command will be executed
    java.io.File f = new java.io.File(sandbox.getSandboxPath());
    if (!f.exists()) {
        throw new GATInvocationException("Unable to find directory " + f.getAbsolutePath());
    }
    // Check and set the environment for a blaunch command
    Map<String, Object> env = sd.getEnvironment();
    this.prepareBLaunchEnv(env);
    // Encapsulate the original command into a blaunch command
    String host = brokerURI.getHost();
    String newExe = this.getLoadLevelerCommand();
    String[] newArgs = this.getLoadLevelerArgs(host, exe, args);
    System.out.println("[AFTER] exe: " + newExe);
    System.out.println("[AFTER] llspawan.stdio args:");
    for (int i = 0; i < newArgs.length; i++) System.out.print(" " + newArgs[i]);
    System.out.println();
    ProcessBundle bundle = new ProcessBundle(description.getProcessCount(), newExe, newArgs, f, env);
    loadlevelerJob.setSubmissionTime();
    loadlevelerJob.setState(Job.JobState.SCHEDULED);
    try {
        loadlevelerJob.setState(Job.JobState.RUNNING);
        loadlevelerJob.waitForTrigger(Job.JobState.RUNNING);
        loadlevelerJob.setStartTime();
        bundle.startBundle();
        loadlevelerJob.setProcess(bundle);
        if (logger.isDebugEnabled()) {
            logger.debug("Job with PID: " + LOADL_PID + " is executed on host " + host);
        }
    } catch (IOException e) {
        throw new CommandNotFoundException("LoadLevelerResourceBrokerAdaptor", e);
    }
    if (!sd.streamingStderrEnabled()) {
        try {
            if (sd.getStderr() != null) {
                OutputStream err = GAT.createFileOutputStream(gatContext, sd.getStderr());
                // to file
                StreamForwarder forwarder = new StreamForwarder(bundle.getStderr(), err, sd.getExecutable() + " [stderr]");
                loadlevelerJob.setErrorStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stderr forwarder to file " + sd.getStderr());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStderr(), null, sd.getExecutable() + " [stderr]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stderr!", e);
        }
    }
    if (!sd.streamingStdoutEnabled()) {
        // read away the stdout
        try {
            if (sd.getStdout() != null) {
                // to file
                OutputStream out = GAT.createFileOutputStream(gatContext, sd.getStdout());
                StreamForwarder forwarder = new StreamForwarder(bundle.getStdout(), out, sd.getExecutable() + " [stdout]");
                loadlevelerJob.setOutputStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stdout forwarder to file " + sd.getStdout());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStdout(), null, sd.getExecutable() + " [stdout]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stdout!", e);
        }
    }
    if (!sd.streamingStdinEnabled() && sd.getStdin() != null) {
        // forward the stdin from file
        try {
            InputStream in = GAT.createFileInputStream(gatContext, sd.getStdin());
            bundle.setStdin(sd.getExecutable(), in);
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file input stream for stdin!", e);
        }
    }
    loadlevelerJob.monitorState();
    return job;
}

Example 3 with Sandbox

use of org.gridlab.gat.resources.cpi.Sandbox in project compss by bsc-wdc.

the class SshLsfResourceBrokerAdaptor method submitJob.

public Job submitJob(AbstractJobDescription abstractDescription, MetricListener listener, String metricDefinitionName) throws GATInvocationException {
    if (!(abstractDescription instanceof JobDescription)) {
        throw new GATInvocationException("can only handle JobDescriptions: " + abstractDescription.getClass());
    }
    JobDescription description = (JobDescription) ((JobDescription) abstractDescription).clone();
    SoftwareDescription sd = description.getSoftwareDescription();
    int nproc = description.getProcessCount();
    if (sd.streamingStderrEnabled() || sd.streamingStdinEnabled() || sd.streamingStdoutEnabled()) {
        throw new GATInvocationException("Streaming I/O not supported by SshLSF adaptor");
    }
    String authority = getAuthority();
    if (authority == null) {
        authority = "localhost";
    }
    // Create filename for return value.
    String returnValueFile = ".rc." + Math.random();
    /*java.io.File jobScriptFile = createJobScript(description);
    	java.io.File jobStarterFile = null;
    	if (nproc > 1) {
    	    jobStarterFile = createJobStarter(description, nproc, jobScriptFile);
    	}*/
    java.io.File bsubFile = createBsubScript(description, returnValueFile, nproc);
    if (logger.isDebugEnabled()) {
        logger.debug("**** Adding pre and post staging:\n  " + sd);
    }
    try {
        // Careful, you may be running on windows, so add a scheme.
        sd.addPostStagedFile(GAT.createFile(gatContext, new URI(returnValueFile)));
    } catch (Throwable e) {
        throw new GATInvocationException("Error in file staging", e);
    }
    ResourceBroker subBroker;
    try {
        subBroker = subResourceBroker(getSubContext(gatContext), brokerURI);
    } catch (Throwable e) {
        throw new GATInvocationException("Could not create subbroker to submit LSF jobs through ssh", e);
    }
    Sandbox sandbox = new Sandbox(gatContext, description, authority, null, true, true, true, true);
    SshLSFJob sshLSFJob = new SshLSFJob(gatContext, brokerURI, description, sandbox, subBroker, returnValueFile);
    Job job = null;
    if (description instanceof WrapperJobDescription) {
        WrapperJobCpi tmp = new WrapperJobCpi(gatContext, sshLSFJob, listener, metricDefinitionName);
        listener = tmp;
        job = tmp;
    } else {
        job = sshLSFJob;
    }
    if (listener != null && metricDefinitionName != null) {
        Metric metric = sshLSFJob.getMetricDefinitionByName(metricDefinitionName).createMetric(null);
        sshLSFJob.addMetricListener(listener, metric);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("*******We are going to perform the ssh submission: ");
    }
    String jobid = sshLsfSubmission(sshLSFJob, description, bsubFile, subBroker, sandbox);
    if (jobid != null) {
        sshLSFJob.setState(Job.JobState.SCHEDULED);
        sshLSFJob.setSoft(description.getSoftwareDescription());
        sshLSFJob.setJobID(jobid);
        sshLSFJob.startListener();
    } else {
        // sandbox.removeSandboxDir();
        throw new GATInvocationException("Could not submit sshLSF job");
    }
    bsubFile.delete();
    return job;
}