Examples with SoftwareDescription org.gridlab.gat.resources.SoftwareDescription used on opensource projects

Example 6 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class SshLsfResourceBrokerAdaptor method sshLsfSubmission.

/*private java.io.File createJobScript(JobDescription description)
			throws GATInvocationException {

		java.io.File temp;

		SoftwareDescription sd = description.getSoftwareDescription();

		try {
			temp = java.io.File.createTempFile("lsf-sub", null);
		} catch (IOException e) {
			throw new GATInvocationException("Cannot create file", e);
		}
		PrintWriter job = null;
		try {
			job = new PrintWriter(new BufferedWriter(new FileWriter(temp)));

			job.print("#!/bin/sh\n");
			job.print("# job script\n");

			// Support DIRECTORY
			String dir = sd.getStringAttribute(SoftwareDescription.DIRECTORY, null);
			if (dir != null) {
				job.print("cd " + dir + "\n");
			}

			// Support environment.
			Map<String, Object> env = sd.getEnvironment();
			if (env != null) {
				Set<String> s = env.keySet();
				Object[] keys = s.toArray();

				for (int i = 0; i < keys.length; i++) {
					String val = (String) env.get(keys[i]);
					job.print(keys[i] + "=" + val + " && export " + keys[i] + "\n");
				}
			}

			// Construct command.
			StringBuffer cmd = new StringBuffer();

			cmd.append(sd.getExecutable().toString());
			if (sd.getArguments() != null) {
				String[] args = sd.getArguments();
				for (int i = 0; i < args.length; ++i) {
					cmd.append(" ");
					cmd.append(args[i]);
				}
			}
			job.print(cmd.toString() + "\n");
			//job.print("exit $?\n");
		} catch (Throwable e) {
			throw new GATInvocationException(
					"Cannot create temporary job script file "
							+ temp.getAbsolutePath(), e);
		} finally {
			if (job != null)
				job.close();
		}
		return temp;
	}*/
private String sshLsfSubmission(SshLSFJob lsfJob, JobDescription description, java.io.File bsubFile, ResourceBroker subBroker, Sandbox sandbox) throws GATInvocationException {
    java.io.File slurmResultFile = null;
    try {
        if (logger.isDebugEnabled()) {
            logger.debug("***** Doing sandbox prestage " + sandbox.getSandboxPath());
        }
        sandbox.prestage();
        if (logger.isDebugEnabled()) {
            logger.debug("***** Sandbox prestage done " + sandbox.getSandboxPath());
        }
        // Create sbatch job
        SoftwareDescription sd = new SoftwareDescription();
        sd.setExecutable("sh");
        sd.setArguments("-c", "bsub < " + bsubFile.getName() + " 2>submit.err");
        // + " && rm -rf " + bsubFile.getName() + " submit.err");
        sd.setAttributes(description.getSoftwareDescription().getAttributes());
        sd.addAttribute(SoftwareDescription.SANDBOX_USEROOT, "true");
        slurmResultFile = java.io.File.createTempFile("GAT", "tmp");
        try {
            sd.setStdout(GAT.createFile(gatContext, new URI(slurmResultFile.toURI())));
            sd.addPreStagedFile(GAT.createFile(gatContext, new URI(bsubFile.toURI())));
        } catch (Throwable e1) {
            try {
                sandbox.removeSandboxDir();
            } catch (Throwable e) {
            // ignore
            }
            throw new GATInvocationException("Could not create GAT object for temporary " + slurmResultFile.getAbsolutePath(), e1);
        }
        // sd.addAttribute(SoftwareDescription.DIRECTORY, sd.getStringAttribute(SoftwareDescription, defaultVal)));
        JobDescription jd = new JobDescription(sd);
        if (logger.isDebugEnabled()) {
            logger.debug("Submitting lsf job: " + sd);
        }
        Job job = subBroker.submitJob(jd, this, "job.status");
        if (logger.isDebugEnabled()) {
            logger.debug("Job submitted.");
        }
        synchronized (job) {
            while (job.getState() != Job.JobState.STOPPED && job.getState() != Job.JobState.SUBMISSION_ERROR) {
                try {
                    job.wait();
                } catch (InterruptedException e) {
                // ignore
                }
            }
        }
        if (job.getState() != Job.JobState.STOPPED || job.getExitStatus() != 0) {
            try {
                sandbox.removeSandboxDir();
            } catch (Throwable e) {
            // ignore
            }
            logger.debug("jobState = " + job.getState() + ", exit status = " + job.getExitStatus());
            throw new GATInvocationException("Could not submit LSF job");
        }
        // submit success.
        BufferedReader in = new BufferedReader(new FileReader(slurmResultFile.getAbsolutePath()));
        String result = in.readLine();
        if (logger.isDebugEnabled()) {
            logger.debug("bsub result line = " + result);
        }
        // Check for LSF bsub result ...
        // TODO Check if LSF return the same
        String job_prefix = "Job <";
        if (result.contains(job_prefix)) {
            int i = result.indexOf(job_prefix);
            result = result.substring(i + job_prefix.length(), result.indexOf(">", i));
        }
        return result;
    } catch (IOException e) {
        try {
            sandbox.removeSandboxDir();
        } catch (Throwable e1) {
        // ignore
        }
        throw new GATInvocationException("Got IOException", e);
    } finally {
        slurmResultFile.delete();
        bsubFile.delete();
    }
}

Example 7 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class SshLsfResourceBrokerAdaptor method createJobStarter.

private java.io.File createJobStarter(JobDescription description, int nproc, java.io.File jobScript) throws GATInvocationException {
    java.io.File temp;
    SoftwareDescription sd = description.getSoftwareDescription();
    try {
        temp = java.io.File.createTempFile("lsf", null);
    } catch (IOException e) {
        throw new GATInvocationException("Cannot create file", e);
    }
    PrintWriter job = null;
    try {
        job = new PrintWriter(new BufferedWriter(new FileWriter(temp)));
        job.print("#!/bin/sh\n");
        job.print("# Job starter script.\n");
        job.print("# The jobs are distributed over the available nodes in round-robin fashion.\n");
        job.print("GAT_MYDIR=`pwd`\n");
        job.print("case X$LSB_HOSTS in\n");
        job.print("X)  GAT_HOSTS=$HOSTNAME\n");
        job.print("    ;;\n");
        job.print("*)  GAT_HOSTS=`cat $LSB_HOSTS | sed 's/ .*//'`\n");
        job.print("    ;;\n");
        job.print("esac\n");
        job.print("GAT_JOBNO=1\n");
        job.print("GAT_JOBS=" + nproc + "\n");
        job.print("set $GAT_HOSTS\n");
        job.print("shift\n");
        job.print("while :\n");
        job.print("do\n");
        job.print("  for GAT_HOST in \"$@\"\n");
        job.print("  do\n");
        job.print("    echo #!/bin/sh > .gat_script.$GAT_JOBNO\n");
        job.print("    echo cd $GAT_MYDIR >> .gat_script.$GAT_JOBNO\n");
        job.print("    echo trap \\\"touch .gat_done.$GAT_JOBNO\\\" 0 1 2 3 15 >> .gat_script.$GAT_JOBNO\n");
        job.print("    cat " + jobScript.getName() + " >> .gat_script.$GAT_JOBNO\n");
        job.print("    chmod +x .gat_script.$GAT_JOBNO\n");
        job.print("    ssh -o StrictHostKeyChecking=false $GAT_HOST \"$GAT_MYDIR/.gat_script.$GAT_JOBNO");
        if (sd.getStdin() != null) {
            job.print(" < $GAT_MYDIR/" + sd.getStdin().getName());
        } else {
            job.print(" < /dev/null");
        }
        job.print(" > $GAT_MYDIR/.out.$GAT_JOBNO 2>$GAT_MYDIR/.err.$GAT_JOBNO &\"\n");
        job.print("    GAT_JOBNO=`expr $GAT_JOBNO + 1`\n");
        job.print("    if expr $GAT_JOBNO \\>= $GAT_JOBS > /dev/null ; then break 2 ; fi\n");
        job.print("  done\n");
        job.print("  set $GAT_HOSTS\n");
        job.print("done\n");
    } catch (Throwable e) {
        throw new GATInvocationException("Cannot create temporary job starter file " + temp.getAbsolutePath(), e);
    } finally {
        if (job != null)
            job.close();
    }
    return temp;
}

Example 8 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class ResourceBrokerAdaptorTest method submitJobStateConsistency.

private AdaptorTestResultEntry submitJobStateConsistency(GATContext gatContext, Preferences preferences, String host) {
    SoftwareDescription sd = new SoftwareDescription();
    sd.setExecutable("/bin/sleep");
    sd.setArguments("2");
    JobDescription jd = new JobDescription(sd);
    ResourceBroker broker;
    try {
        broker = GAT.createResourceBroker(gatContext, preferences, new URI(host));
    } catch (GATObjectCreationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    } catch (URISyntaxException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    JobStateMetricListener listener = new JobStateMetricListener(this);
    long start = System.currentTimeMillis();
    try {
        broker.submitJob(jd, listener, "job.status");
    } catch (GATInvocationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    waitForJob();
    long stop = System.currentTimeMillis();
    return new AdaptorTestResultEntry(listener.getException() == null, (stop - start), listener.getException());
}

Example 9 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class ResourceBrokerAdaptorTest method submitJobEnvironment.

private AdaptorTestResultEntry submitJobEnvironment(GATContext gatContext, Preferences preferences, String host) {
    SoftwareDescription sd = new SoftwareDescription();
    Map<String, Object> env = new HashMap<String, Object>();
    env.put("JAVAGAT_TEST_KEY", "javagat-test-value");
    sd.setEnvironment(env);
    sd.setExecutable("/usr/bin/env");
    try {
        sd.setStdout(GAT.createFile(gatContext, preferences, "stdout"));
    } catch (GATObjectCreationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    JobDescription jd = new JobDescription(sd);
    ResourceBroker broker;
    try {
        broker = GAT.createResourceBroker(gatContext, preferences, new URI(host));
    } catch (GATObjectCreationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    } catch (URISyntaxException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    long start = System.currentTimeMillis();
    Job job;
    try {
        job = broker.submitJob(jd, this, "job.status");
    } catch (GATInvocationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    waitForJob();
    try {
        Map<String, Object> info = job.getInfo();
        Throwable ex = (Throwable) info.get("poststage.exception");
        if (ex != null) {
            return new AdaptorTestResultEntry(false, 0L, ex);
        }
    } catch (GATInvocationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    long stop = System.currentTimeMillis();
    boolean success = false;
    try {
        BufferedReader reader = new BufferedReader(new InputStreamReader(new java.io.FileInputStream("stdout")));
        while (true) {
            String result = reader.readLine();
            if (result == null) {
                break;
            }
            if (result.contains("JAVAGAT_TEST_KEY") && result.contains("javagat-test-value")) {
                success = true;
            }
        }
        reader.close();
    } catch (Exception e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    return new AdaptorTestResultEntry(success, (stop - start), null);
}

Example 10 with SoftwareDescription

use of org.gridlab.gat.resources.SoftwareDescription in project compss by bsc-wdc.

the class ResourceBrokerAdaptorTest method submitJobGetInfo.

private AdaptorTestResultEntry submitJobGetInfo(GATContext gatContext, Preferences preferences, String host) {
    SoftwareDescription sd = new SoftwareDescription();
    sd.setExecutable("/bin/sleep");
    sd.setArguments("2");
    JobDescription jd = new JobDescription(sd);
    ResourceBroker broker;
    try {
        broker = GAT.createResourceBroker(gatContext, preferences, new URI(host));
    } catch (GATObjectCreationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    } catch (URISyntaxException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    Job job = null;
    Exception exception = null;
    long start = System.currentTimeMillis();
    try {
        job = broker.submitJob(jd);
    } catch (GATInvocationException e) {
        return new AdaptorTestResultEntry(false, 0L, e);
    }
    while (job.getState() != Job.JobState.STOPPED) {
        Map<String, Object> info = null;
        try {
            info = job.getInfo();
        } catch (GATInvocationException e) {
            exception = e;
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e1) {
            // ignored
            }
            continue;
        }
        if (info == null) {
            exception = new Exception("getInfo returns null");
        } else {
            if (!info.containsKey("state")) {
                exception = new Exception("getInfo doesn't contain a key 'state'");
            }
            if (!info.containsKey("hostname")) {
                exception = new Exception("getInfo doesn't contain a key 'hostname'");
            } else {
                if (info.get("state").equals(Job.JobState.RUNNING) && info.get("hostname") == null) {
                    exception = new Exception("inconsistent getInfo: state=RUNNING, hostname=null");
                }
            }
            if (!info.containsKey("submissiontime")) {
                exception = new Exception("getInfo doesn't contain a key 'submissiontime'");
            }
            if (!info.containsKey("starttime")) {
                exception = new Exception("getInfo doesn't contain a key 'starttime'");
            }
            if (!info.containsKey("stoptime")) {
                exception = new Exception("getInfo doesn't contain a key 'stoptime'");
            }
            if (!info.containsKey("poststage.exception")) {
                exception = new Exception("getInfo doesn't contain a key 'poststage.exception'");
            }
        }
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            exception = e;
        }
    }
    long stop = System.currentTimeMillis();
    return new AdaptorTestResultEntry(exception == null, (stop - start), exception);
}