use of org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager in project airavata by apache.
the class ForkJobSubmissionTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
try {
ProcessContext processContext = taskContext.getParentProcessContext();
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
if (resourceJobManager != null) {
jConfig = Factory.getJobManagerConfiguration(resourceJobManager);
}
JobStatus jobStatus = new JobStatus();
File jobFile = GFacUtils.createJobFile(groovyMap, taskContext, jConfig);
if (jobFile != null && jobFile.exists()) {
jobModel.setJobDescription(FileUtils.readFileToString(jobFile));
JobSubmissionOutput jobSubmissionOutput = remoteCluster.submitBatchJob(jobFile.getPath(), processContext.getWorkingDir());
jobModel.setExitCode(jobSubmissionOutput.getExitCode());
jobModel.setStdErr(jobSubmissionOutput.getStdErr());
jobModel.setStdOut(jobSubmissionOutput.getStdOut());
String jobId = jobSubmissionOutput.getJobId();
if (jobId != null && !jobId.isEmpty()) {
jobModel.setJobId(jobId);
GFacUtils.saveJobModel(processContext, jobModel);
jobStatus.setJobState(JobState.SUBMITTED);
jobStatus.setReason("Successfully Submitted to " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
taskStatus = new TaskStatus(TaskState.COMPLETED);
taskStatus.setReason("Submitted job to compute resource");
}
if (jobId == null || jobId.isEmpty()) {
String msg = "expId:" + processContext.getProcessModel().getExperimentId() + " Couldn't find " + "remote jobId for JobName:" + jobModel.getJobName() + ", both submit and verify steps " + "doesn't return a valid JobId. " + "Hence changing experiment state to Failed";
log.error(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(msg);
errorModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
GFacUtils.saveExperimentError(processContext, errorModel);
GFacUtils.saveProcessError(processContext, errorModel);
GFacUtils.saveTaskError(taskContext, errorModel);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason("Couldn't find job id in both submitted and verified steps");
} else {
GFacUtils.saveJobModel(processContext, jobModel);
}
} else {
taskStatus.setState(TaskState.FAILED);
if (jobFile == null) {
taskStatus.setReason("JobFile is null");
} else {
taskStatus.setReason("Job file doesn't exist");
}
}
} catch (ApplicationSettingsException e) {
String msg = "Error occurred while creating job descriptor";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (AppCatalogException e) {
String msg = "Error while instantiating app catalog";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (GFacException e) {
String msg = "Error occurred while submitting the job";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (IOException e) {
String msg = "Error while reading the content of the job file";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
}
return taskStatus;
}
use of org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager in project airavata by apache.
the class LocalJobSubmissionTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
try {
ProcessContext processContext = taskContext.getParentProcessContext();
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
String jobId = AiravataUtils.getId("JOB_ID_");
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
jobModel.setJobId(jobId);
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
if (resourceJobManager != null) {
jConfig = Factory.getJobManagerConfiguration(resourceJobManager);
}
JobStatus jobStatus = new JobStatus();
File jobFile = GFacUtils.createJobFile(groovyMap, taskContext, jConfig);
if (jobFile != null && jobFile.exists()) {
jobModel.setJobDescription(FileUtils.readFileToString(jobFile));
GFacUtils.saveJobModel(processContext, jobModel);
JobSubmissionOutput jobSubmissionOutput = remoteCluster.submitBatchJob(jobFile.getPath(), processContext.getWorkingDir());
jobStatus.setJobState(JobState.SUBMITTED);
jobStatus.setReason("Successfully Submitted to " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
// log job submit status
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
// for local, job gets completed synchronously
// so changing job status to complete
jobModel.setExitCode(jobSubmissionOutput.getExitCode());
jobModel.setStdErr(jobSubmissionOutput.getStdErr());
jobModel.setStdOut(jobSubmissionOutput.getStdOut());
jobModel.setJobId(jobId);
jobStatus.setJobState(JobState.COMPLETE);
jobStatus.setReason("Successfully Completed " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
// log job complete status
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
taskStatus = new TaskStatus(TaskState.COMPLETED);
taskStatus.setReason("Submitted job to compute resource");
} else {
taskStatus.setState(TaskState.FAILED);
if (jobFile == null) {
taskStatus.setReason("JobFile is null");
} else {
taskStatus.setReason("Job file doesn't exist");
}
}
} catch (GFacException | IOException | AppCatalogException | ApplicationSettingsException e) {
String msg = "Error occurred while submitting a local job";
log.error(msg, e);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
taskStatus.setState(TaskState.FAILED);
}
return taskStatus;
}
use of org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager in project airavata by apache.
the class DocumentCreatorNew method createStampedeAmberDocs.
public String createStampedeAmberDocs() throws InvalidRequestException, AiravataClientException, AiravataSystemException, TException, AppCatalogException {
ComputeResourceDescription host = DocumentCreatorUtils.createComputeResourceDescription(stampedeHostAddress, null, null);
host.addToHostAliases(stampedeHostAddress);
host.addToIpAddresses(stampedeHostAddress);
host.setComputeResourceId(client.registerComputeResource(authzToken, host));
ResourceJobManager resourceJobManager = DocumentCreatorUtils.createResourceJobManager(ResourceJobManagerType.SLURM, "/usr/bin/", null, "push");
SSHJobSubmission sshJobSubmission = new SSHJobSubmission();
sshJobSubmission.setResourceJobManager(resourceJobManager);
sshJobSubmission.setSecurityProtocol(SecurityProtocol.GSI);
sshJobSubmission.setSshPort(2222);
client.addSSHJobSubmissionDetails(authzToken, host.getComputeResourceId(), 1, sshJobSubmission);
SCPDataMovement scpDataMovement = new SCPDataMovement();
scpDataMovement.setSecurityProtocol(SecurityProtocol.GSI);
scpDataMovement.setSshPort(22);
client.addSCPDataMovementDetails(authzToken, host.getComputeResourceId(), 1, scpDataMovement);
ApplicationModule amodule = DocumentCreatorUtils.createApplicationModule("Amber", "12.0", null);
amodule.setAppModuleId(client.registerApplicationModule(authzToken, DEFAULT_GATEWAY, amodule));
ApplicationInterfaceDescription application = new ApplicationInterfaceDescription();
application.setApplicationName("AmberBR2");
application.addToApplicationModules(amodule.getAppModuleId());
application.addToApplicationInputs(DocumentCreatorUtils.createAppInput("AMBER_HEAT_RST", "AMBER_HEAT_RST", null, null, DataType.URI));
application.addToApplicationInputs(DocumentCreatorUtils.createAppInput("AMBER_PROD_IN", "AMBER_PROD_IN", null, null, DataType.URI));
application.addToApplicationInputs(DocumentCreatorUtils.createAppInput("AMBER_PRMTOP", "AMBER_PRMTOP", null, null, DataType.URI));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("AMBER_Prod.info", null, DataType.URI));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("AMBER_Prod.mdcrd", null, DataType.URI));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("AMBER_Prod.out", null, DataType.URI));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("AMBER_Prod.rst", null, DataType.URI));
application.setApplicationInterfaceId(client.registerApplicationInterface(authzToken, DEFAULT_GATEWAY, application));
ApplicationDeploymentDescription deployment = DocumentCreatorUtils.createApplicationDeployment(host.getComputeResourceId(), amodule.getAppModuleId(), "/home1/01437/ogce/production/app_wrappers/amber_wrapper.sh", ApplicationParallelismType.SERIAL, "AmberStampede");
deployment.setAppDeploymentId(client.registerApplicationDeployment(authzToken, DEFAULT_GATEWAY, deployment));
client.addGatewayComputeResourcePreference(authzToken, getGatewayResourceProfile().getGatewayID(), host.getComputeResourceId(), DocumentCreatorUtils.createComputeResourcePreference(host.getComputeResourceId(), "/home1/01437/ogce", "TG-STA110014S", false, null, null, null));
return host.getComputeResourceId() + "," + application.getApplicationInterfaceId();
}
use of org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager in project airavata by apache.
the class DocumentCreatorNew method createLocalHostDocs.
public String createLocalHostDocs() throws AppCatalogException, InvalidRequestException, AiravataClientException, AiravataSystemException, TException {
// Define compute resource host
ComputeResourceDescription host = DocumentCreatorUtils.createComputeResourceDescription("localhost", new ArrayList<String>(Arrays.asList(new String[] { "127.0.0.1" })), new ArrayList<String>(Arrays.asList(new String[] { "127.0.0.1" })));
// host.setIsEmpty(true);
host.setComputeResourceId(client.registerComputeResource(authzToken, host));
LOCALSubmission localSubmission = new LOCALSubmission();
ResourceJobManager resourceJobManager = DocumentCreatorUtils.createResourceJobManager(ResourceJobManagerType.FORK, null, null, null);
localSubmission.setResourceJobManager(resourceJobManager);
client.addLocalSubmissionDetails(authzToken, host.getComputeResourceId(), 1, localSubmission);
LOCALDataMovement localDataMovement = new LOCALDataMovement();
client.addLocalDataMovementDetails(authzToken, host.getComputeResourceId(), 1, localDataMovement);
// Define application module
ApplicationModule module = DocumentCreatorUtils.createApplicationModule("echo", "1.0.0", "Local host echo applications");
module.setAppModuleId(client.registerApplicationModule(authzToken, DEFAULT_GATEWAY, module));
// Define application interfaces
ApplicationInterfaceDescription application = new ApplicationInterfaceDescription();
// application.setIsEmpty(false);
application.setApplicationName("SimpleEcho0");
application.addToApplicationModules(module.getAppModuleId());
application.addToApplicationInputs(DocumentCreatorUtils.createAppInput("echo_input", "echo_input", "Echo Input Data", null, DataType.STRING));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("echo_output", null, DataType.STRING));
application.setApplicationInterfaceId(client.registerApplicationInterface(authzToken, DEFAULT_GATEWAY, application));
// Define application deployment
ApplicationDeploymentDescription deployment = DocumentCreatorUtils.createApplicationDeployment(host.getComputeResourceId(), module.getAppModuleId(), "/bin/echo", ApplicationParallelismType.SERIAL, "Local echo app depoyment");
deployment.setAppDeploymentId(client.registerApplicationDeployment(authzToken, DEFAULT_GATEWAY, deployment));
// Define gateway profile
ComputeResourcePreference computeResourcePreference = DocumentCreatorUtils.createComputeResourcePreference(host.getComputeResourceId(), "/tmp", null, false, null, null, null);
gatewayResourceProfile = new GatewayResourceProfile();
// gatewayResourceProfile.setGatewayID("default");
gatewayResourceProfile.setGatewayID(DEFAULT_GATEWAY);
gatewayResourceProfile.addToComputeResourcePreferences(computeResourcePreference);
String gatewayId = client.registerGatewayResourceProfile(authzToken, gatewayResourceProfile);
gatewayResourceProfile.setGatewayID(gatewayId);
client.addGatewayComputeResourcePreference(authzToken, gatewayResourceProfile.getGatewayID(), host.getComputeResourceId(), computeResourcePreference);
return host.getComputeResourceId() + "," + application.getApplicationInterfaceId();
}
use of org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager in project airavata by apache.
the class DocumentCreatorNew method createPBSDocsForOGCE_Echo.
//
// public void createGramDocs() {
// // /*
// // creating host descriptor for gram
// // */
// // HostDescription host = new HostDescription(GlobusHostType.type);
// // host.getType().setHostAddress(trestleshpcHostAddress);
// // host.getType().setHostName(trestleshpcHostAddress);
// // ((GlobusHostType) host.getType()).setGlobusGateKeeperEndPointArray(new String[]{gramAddress});
// // ((GlobusHostType) host.getType()).setGridFTPEndPointArray(new String[]{gridftpAddress});
// // try {
// // airavataAPI.getApplicationManager().saveHostDescription(host);
// // } catch (AiravataAPIInvocationException e) {
// // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
// // }
// //
// //
// // /*
// // * Service Description creation and saving
// // */
// // String serviceName = "SimpleEcho1";
// // ServiceDescription serv = new ServiceDescription();
// // serv.getType().setName(serviceName);
// //
// // List<InputParameterType> inputList = new ArrayList<InputParameterType>();
// // List<OutputParameterType> outputList = new ArrayList<OutputParameterType>();
// //
// // InputParameterType input = InputParameterType.Factory.newInstance();
// // input.setParameterName("echo_input");
// // ParameterType parameterType = input.addNewParameterType();
// // parameterType.setType(DataType.STRING);
// // parameterType.setName("String");
// //
// // OutputParameterType output = OutputParameterType.Factory.newInstance();
// // output.setParameterName("echo_output");
// // ParameterType parameterType1 = output.addNewParameterType();
// // parameterType1.setType(DataType.STRING);
// // parameterType1.setName("String");
// //
// // inputList.add(input);
// // outputList.add(output);
// //
// // InputParameterType[] inputParamList = inputList.toArray(new InputParameterType[inputList.size()]);
// // OutputParameterType[] outputParamList = outputList.toArray(new OutputParameterType[outputList.size()]);
// //
// // serv.getType().setInputParametersArray(inputParamList);
// // serv.getType().setOutputParametersArray(outputParamList);
// // try {
// // airavataAPI.getApplicationManager().saveServiceDescription(serv);
// // } catch (AiravataAPIInvocationException e) {
// // e.printStackTrace();
// // }
// //
// // /*
// // Application descriptor creation and saving
// // */
// // ApplicationDescription appDesc = new ApplicationDescription(HpcApplicationDeploymentType.type);
// // HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) appDesc.getType();
// // ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory.newInstance();
// // name.setStringValue("EchoLocal");
// // app.setApplicationName(name);
// // ProjectAccountType projectAccountType = app.addNewProjectAccount();
// // projectAccountType.setProjectAccountNumber("sds128");
// //
// // QueueType queueType = app.addNewQueue();
// // queueType.setQueueName("normal");
// //
// // app.setCpuCount(1);
// // app.setJobType(JobTypeType.SERIAL);
// // app.setNodeCount(1);
// // app.setProcessorsPerNode(1);
// //
// // /*
// // * Use bat file if it is compiled on Windows
// // */
// // app.setExecutableLocation("/bin/echo");
// //
// // /*
// // * Default tmp location
// // */
// // String tempDir = "/home/ogce/scratch";
// // app.setScratchWorkingDirectory(tempDir);
// // app.setMaxMemory(10);
// //
// //
// // try {
// // airavataAPI.getApplicationManager().saveApplicationDescription(serviceName, trestleshpcHostAddress, appDesc);
// // } catch (AiravataAPIInvocationException e) {
// // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
// // }
// }
//
public String createPBSDocsForOGCE_Echo() throws AppCatalogException, InvalidRequestException, AiravataClientException, AiravataSystemException, TException {
ComputeResourceDescription host = DocumentCreatorUtils.createComputeResourceDescription(trestleshpcHostAddress, null, null);
host.addToIpAddresses(trestleshpcHostAddress);
host.addToHostAliases(trestleshpcHostAddress);
host.setComputeResourceId(client.registerComputeResource(authzToken, host));
SSHJobSubmission sshJobSubmission = new SSHJobSubmission();
ResourceJobManager resourceJobManager = DocumentCreatorUtils.createResourceJobManager(ResourceJobManagerType.PBS, "/opt/torque/bin/", null, null);
sshJobSubmission.setResourceJobManager(resourceJobManager);
sshJobSubmission.setSecurityProtocol(SecurityProtocol.GSI);
sshJobSubmission.setSshPort(22);
client.addSSHJobSubmissionDetails(authzToken, host.getComputeResourceId(), 1, sshJobSubmission);
SCPDataMovement scpDataMovement = new SCPDataMovement();
scpDataMovement.setSecurityProtocol(SecurityProtocol.GSI);
scpDataMovement.setSshPort(22);
client.addSCPDataMovementDetails(authzToken, host.getComputeResourceId(), 1, scpDataMovement);
ApplicationModule module1 = DocumentCreatorUtils.createApplicationModule("echo", "1.2", null);
module1.setAppModuleId(client.registerApplicationModule(authzToken, DEFAULT_GATEWAY, module1));
ApplicationInterfaceDescription application = new ApplicationInterfaceDescription();
// application.setIsEmpty(false);
application.setApplicationName("SimpleEcho2");
application.addToApplicationModules(module1.getAppModuleId());
application.addToApplicationInputs(DocumentCreatorUtils.createAppInput("echo_input", "echo_input", "echo_input", null, DataType.STRING));
application.addToApplicationOutputs(DocumentCreatorUtils.createAppOutput("echo_output", null, DataType.STRING));
application.setApplicationInterfaceId(client.registerApplicationInterface(authzToken, DEFAULT_GATEWAY, application));
ApplicationDeploymentDescription deployment = DocumentCreatorUtils.createApplicationDeployment(host.getComputeResourceId(), module1.getAppModuleId(), "/home/ogce/echo.sh", ApplicationParallelismType.SERIAL, "Echo application");
deployment.setAppDeploymentId(client.registerApplicationDeployment(authzToken, DEFAULT_GATEWAY, deployment));
client.addGatewayComputeResourcePreference(authzToken, getGatewayResourceProfile().getGatewayID(), host.getComputeResourceId(), DocumentCreatorUtils.createComputeResourcePreference(host.getComputeResourceId(), "/oasis/scratch/trestles/ogce/temp_project/", "sds128", false, null, null, null));
return host.getComputeResourceId() + "," + application.getApplicationInterfaceId();
}
Aggregations