Search in sources :

Example 21 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class LoadLevelerResourceBrokerAdaptor method submitJob.

/*
     * (non-Javadoc)
     * 
     * @see org.gridlab.gat.resources.ResourceBroker#submitJob(org.gridlab.gat.resources.JobDescription)
     */
public Job submitJob(AbstractJobDescription abstractDescription, MetricListener listener, String metricDefinitionName) throws GATInvocationException {
    if (!(abstractDescription instanceof JobDescription)) {
        throw new GATInvocationException("can only handle JobDescriptions: " + abstractDescription.getClass());
    }
    JobDescription description = (JobDescription) abstractDescription;
    SoftwareDescription sd = description.getSoftwareDescription();
    if (sd == null) {
        throw new GATInvocationException("The job description does not contain a software description");
    }
    if (description.getProcessCount() < 1) {
        throw new GATInvocationException("Adaptor cannot handle: process count < 1: " + description.getProcessCount());
    }
    if (description.getResourceCount() != 1) {
        throw new GATInvocationException("Adaptor cannot handle: resource count > 1: " + description.getResourceCount());
    }
    String home = System.getProperty("user.home");
    if (home == null) {
        throw new GATInvocationException("loadleveler broker could not get user home dir");
    }
    Sandbox sandbox = new Sandbox(gatContext, description, "localhost", home, true, true, false, false);
    LoadLevelerJob loadlevelerJob = new LoadLevelerJob(gatContext, description, sandbox);
    Job job = null;
    if (description instanceof WrapperJobDescription) {
        WrapperJobCpi tmp = new WrapperJobCpi(gatContext, loadlevelerJob, listener, metricDefinitionName);
        listener = tmp;
        job = tmp;
    } else {
        job = loadlevelerJob;
    }
    if (listener != null && metricDefinitionName != null) {
        Metric metric = loadlevelerJob.getMetricDefinitionByName(metricDefinitionName).createMetric(null);
        loadlevelerJob.addMetricListener(listener, metric);
    }
    loadlevelerJob.setState(Job.JobState.PRE_STAGING);
    loadlevelerJob.waitForTrigger(Job.JobState.PRE_STAGING);
    sandbox.prestage();
    String exe;
    if (sandbox.getResolvedExecutable() != null) {
        exe = sandbox.getResolvedExecutable().getPath();
    // try to set the executable bit, it might be lost
    /* CDIAZ: The command "exe" can be also in a remote host
             * 		  The command must have the right permissions in the remote host
            try {
                new CommandRunner("chmod", "+x", exe);
            } catch (Throwable t) {
                // ignore
            }
            */
    } else {
        exe = getExecutable(description);
    }
    String[] args = getArgumentsArray(description);
    // Directory where the loadleveler command will be executed
    java.io.File f = new java.io.File(sandbox.getSandboxPath());
    if (!f.exists()) {
        throw new GATInvocationException("Unable to find directory " + f.getAbsolutePath());
    }
    // Check and set the environment for a blaunch command
    Map<String, Object> env = sd.getEnvironment();
    this.prepareBLaunchEnv(env);
    // Encapsulate the original command into a blaunch command
    String host = brokerURI.getHost();
    String newExe = this.getLoadLevelerCommand();
    String[] newArgs = this.getLoadLevelerArgs(host, exe, args);
    System.out.println("[AFTER] exe: " + newExe);
    System.out.println("[AFTER] llspawan.stdio args:");
    for (int i = 0; i < newArgs.length; i++) System.out.print(" " + newArgs[i]);
    System.out.println();
    ProcessBundle bundle = new ProcessBundle(description.getProcessCount(), newExe, newArgs, f, env);
    loadlevelerJob.setSubmissionTime();
    loadlevelerJob.setState(Job.JobState.SCHEDULED);
    try {
        loadlevelerJob.setState(Job.JobState.RUNNING);
        loadlevelerJob.waitForTrigger(Job.JobState.RUNNING);
        loadlevelerJob.setStartTime();
        bundle.startBundle();
        loadlevelerJob.setProcess(bundle);
        if (logger.isDebugEnabled()) {
            logger.debug("Job with PID: " + LOADL_PID + " is executed on host " + host);
        }
    } catch (IOException e) {
        throw new CommandNotFoundException("LoadLevelerResourceBrokerAdaptor", e);
    }
    if (!sd.streamingStderrEnabled()) {
        try {
            if (sd.getStderr() != null) {
                OutputStream err = GAT.createFileOutputStream(gatContext, sd.getStderr());
                // to file
                StreamForwarder forwarder = new StreamForwarder(bundle.getStderr(), err, sd.getExecutable() + " [stderr]");
                loadlevelerJob.setErrorStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stderr forwarder to file " + sd.getStderr());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStderr(), null, sd.getExecutable() + " [stderr]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stderr!", e);
        }
    }
    if (!sd.streamingStdoutEnabled()) {
        // read away the stdout
        try {
            if (sd.getStdout() != null) {
                // to file
                OutputStream out = GAT.createFileOutputStream(gatContext, sd.getStdout());
                StreamForwarder forwarder = new StreamForwarder(bundle.getStdout(), out, sd.getExecutable() + " [stdout]");
                loadlevelerJob.setOutputStream(forwarder);
                if (logger.isDebugEnabled()) {
                    logger.debug("Created stdout forwarder to file " + sd.getStdout());
                }
            } else {
                // or throw it away
                new StreamForwarder(bundle.getStdout(), null, sd.getExecutable() + " [stdout]");
            }
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file output stream for stdout!", e);
        }
    }
    if (!sd.streamingStdinEnabled() && sd.getStdin() != null) {
        // forward the stdin from file
        try {
            InputStream in = GAT.createFileInputStream(gatContext, sd.getStdin());
            bundle.setStdin(sd.getExecutable(), in);
        } catch (GATObjectCreationException e) {
            throw new GATInvocationException("Unable to create file input stream for stdin!", e);
        }
    }
    loadlevelerJob.monitorState();
    return job;
}
Also used : ProcessBundle(org.gridlab.gat.engine.util.ProcessBundle) OutputStream(java.io.OutputStream) StreamForwarder(org.gridlab.gat.engine.util.StreamForwarder) Sandbox(org.gridlab.gat.resources.cpi.Sandbox) WrapperJobDescription(org.gridlab.gat.resources.WrapperJobDescription) JobDescription(org.gridlab.gat.resources.JobDescription) AbstractJobDescription(org.gridlab.gat.resources.AbstractJobDescription) Job(org.gridlab.gat.resources.Job) WrapperJobDescription(org.gridlab.gat.resources.WrapperJobDescription) GATObjectCreationException(org.gridlab.gat.GATObjectCreationException) InputStream(java.io.InputStream) IOException(java.io.IOException) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription) GATInvocationException(org.gridlab.gat.GATInvocationException) WrapperJobCpi(org.gridlab.gat.resources.cpi.WrapperJobCpi) Metric(org.gridlab.gat.monitoring.Metric) CommandNotFoundException(org.gridlab.gat.CommandNotFoundException)

Example 22 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class SshLSFJob method kill.

protected void kill(String jobID) {
    try {
        // Create qdel job
        SoftwareDescription sd = new SoftwareDescription();
        sd.setExecutable("bkill");
        sd.setArguments(jobID);
        sd.addAttribute(SoftwareDescription.SANDBOX_USEROOT, "true");
        sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, sandbox.getSandboxPath());
        sd.addAttribute(SoftwareDescription.STOP_ON_EXIT, "false");
        JobDescription jd = new JobDescription(sd);
        Job job = jobHelper.submitJob(jd, this, "job.status");
        synchronized (job) {
            while (job.getState() != Job.JobState.STOPPED && job.getState() != Job.JobState.SUBMISSION_ERROR) {
                try {
                    job.wait();
                } catch (InterruptedException e) {
                // ignore
                }
            }
        }
        if (job.getState() != Job.JobState.STOPPED || job.getExitStatus() != 0) {
            throw new GATInvocationException("Could not submit scancel job");
        }
    } catch (Throwable e) {
        logger.info("Failed to stop sshSlurm job: " + jobID, e);
    }
}
Also used : JobDescription(org.gridlab.gat.resources.JobDescription) GATInvocationException(org.gridlab.gat.GATInvocationException) SshLSFJob(org.gridlab.gat.resources.cpi.sshlsf.SshLSFJob) Job(org.gridlab.gat.resources.Job) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription)

Example 23 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class AdvertJob method main.

public static void main(String[] args) throws Exception {
    try {
        GATContext c = new GATContext();
        Preferences prefs = new Preferences();
        prefs.put("File.adaptor.name", "local,commandlinessh");
        prefs.put("job.stop.on.exit", "false");
        c.addPreferences(prefs);
        SoftwareDescription sd = new SoftwareDescription();
        sd.setExecutable("/bin/sleep");
        sd.setArguments("100");
        // stdout & stderr
        File stdout = GAT.createFile(c, "std.out");
        File stderr = GAT.createFile(c, "std.err");
        sd.setStderr(stderr);
        sd.setStdout(stdout);
        ResourceDescription rd = new HardwareResourceDescription();
        JobDescription jd = new JobDescription(sd, rd);
        ResourceBroker broker = GAT.createResourceBroker(c, new URI("sshsge://fs0.das3.cs.vu.nl"));
        Job job = broker.submitJob(jd);
        AdvertService a = GAT.createAdvertService(c);
        MetaData m = new MetaData();
        m.put("name", "testJob");
        a.add(job, m, "/rob/testJob");
        a.exportDataBase(new URI("file:///mydb"));
        GAT.end();
        System.exit(0);
    } catch (Throwable e) {
        e.printStackTrace();
    }
}
Also used : GATContext(org.gridlab.gat.GATContext) ResourceBroker(org.gridlab.gat.resources.ResourceBroker) HardwareResourceDescription(org.gridlab.gat.resources.HardwareResourceDescription) AdvertService(org.gridlab.gat.advert.AdvertService) URI(org.gridlab.gat.URI) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription) JobDescription(org.gridlab.gat.resources.JobDescription) ResourceDescription(org.gridlab.gat.resources.ResourceDescription) HardwareResourceDescription(org.gridlab.gat.resources.HardwareResourceDescription) MetaData(org.gridlab.gat.advert.MetaData) Preferences(org.gridlab.gat.Preferences) Job(org.gridlab.gat.resources.Job) File(org.gridlab.gat.io.File)

Example 24 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class FileInputStreamAdaptorTest method run.

private void run(String host, String script) {
    Preferences preferences = new Preferences();
    preferences.put("resourcebroker.adaptor.name", "commandlinessh,sshtrilead,local");
    preferences.put("file.adaptor.name", "commandlinessh,sshtrilead,local");
    SoftwareDescription sd = new SoftwareDescription();
    sd.setExecutable("/bin/bash");
    sd.setArguments(script);
    try {
        sd.addPreStagedFile(GAT.createFile(preferences, "tests" + java.io.File.separator + "src" + java.io.File.separator + "benchmarks" + java.io.File.separator + script));
    } catch (GATObjectCreationException e) {
        e.printStackTrace();
        System.exit(1);
    }
    ResourceBroker broker = null;
    try {
        broker = GAT.createResourceBroker(preferences, new URI("any://" + host));
    } catch (GATObjectCreationException e) {
        e.printStackTrace();
        System.exit(1);
    } catch (URISyntaxException e) {
        e.printStackTrace();
        System.exit(1);
    }
    Job job = null;
    try {
        job = broker.submitJob(new JobDescription(sd));
    } catch (GATInvocationException e) {
        e.printStackTrace();
        System.exit(1);
    }
    while (job.getState() != Job.JobState.STOPPED && job.getState() != Job.JobState.SUBMISSION_ERROR) {
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
        // ignored
        }
    }
}
Also used : JobDescription(org.gridlab.gat.resources.JobDescription) GATObjectCreationException(org.gridlab.gat.GATObjectCreationException) GATInvocationException(org.gridlab.gat.GATInvocationException) ResourceBroker(org.gridlab.gat.resources.ResourceBroker) URISyntaxException(java.net.URISyntaxException) Preferences(org.gridlab.gat.Preferences) Job(org.gridlab.gat.resources.Job) URI(org.gridlab.gat.URI) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription)

Example 25 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class ResourceBrokerAdaptorTest method submitJobEasy.

private AdaptorTestResultEntry submitJobEasy(GATContext gatContext, Preferences preferences, String host) {
    SoftwareDescription sd = new SoftwareDescription();
    sd.setExecutable("/bin/echo");
    sd.setArguments("test", "1", "2", "3");
    Map<String, Object> attributes = new HashMap<String, Object>();
    sd.setAttributes(attributes);
    JobDescription jd = new JobDescription(sd);
    ResourceBroker broker;
    try {
        broker = GAT.createResourceBroker(gatContext, preferences, new URI(host));
    } catch (GATObjectCreationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    } catch (URISyntaxException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    long start = System.currentTimeMillis();
    try {
        broker.submitJob(jd, this, "job.status");
    } catch (GATInvocationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    waitForJob();
    long stop = System.currentTimeMillis();
    return new AdaptorTestResultEntry(true, (stop - start), null);
}
Also used : GATObjectCreationException(org.gridlab.gat.GATObjectCreationException) HashMap(java.util.HashMap) ResourceBroker(org.gridlab.gat.resources.ResourceBroker) URISyntaxException(java.net.URISyntaxException) URI(org.gridlab.gat.URI) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription) JobDescription(org.gridlab.gat.resources.JobDescription) GATInvocationException(org.gridlab.gat.GATInvocationException)

Aggregations

SoftwareDescription (org.gridlab.gat.resources.SoftwareDescription)34 JobDescription (org.gridlab.gat.resources.JobDescription)31 URI (org.gridlab.gat.URI)27 GATInvocationException (org.gridlab.gat.GATInvocationException)24 Job (org.gridlab.gat.resources.Job)24 ResourceBroker (org.gridlab.gat.resources.ResourceBroker)24 GATObjectCreationException (org.gridlab.gat.GATObjectCreationException)17 URISyntaxException (java.net.URISyntaxException)16 IOException (java.io.IOException)11 BufferedReader (java.io.BufferedReader)8 Preferences (org.gridlab.gat.Preferences)8 File (org.gridlab.gat.io.File)8 File (java.io.File)6 WrapperJobDescription (org.gridlab.gat.resources.WrapperJobDescription)5 FileReader (java.io.FileReader)4 InputStreamReader (java.io.InputStreamReader)4 AbstractJobDescription (org.gridlab.gat.resources.AbstractJobDescription)4 InputStream (java.io.InputStream)3 HashMap (java.util.HashMap)3 GATContext (org.gridlab.gat.GATContext)3