use of org.apache.airavata.model.status.JobStatus in project airavata by apache.
the class GFacEngineImpl method executeCancel.
private void executeCancel(TaskContext taskContext, JobSubmissionTask jSTask) throws GFacException {
try {
JobStatus oldJobStatus = jSTask.cancel(taskContext);
// If Job was in Queued state when cancel command runs, then there won't be any email from this job.
ProcessContext pc = taskContext.getParentProcessContext();
JobMonitor monitorService = Factory.getMonitorService(pc.getMonitorMode());
monitorService.canceledJob(pc.getJobModel().getJobId());
} catch (TaskException e) {
throw new GFacException("Error while cancelling job");
} catch (AiravataException e) {
throw new GFacException("Error wile getting monitoring service");
}
}
use of org.apache.airavata.model.status.JobStatus in project airavata by apache.
the class GFacEngineImpl method executeTaskListFrom.
private void executeTaskListFrom(ProcessContext processContext, String startingTaskId) throws GFacException {
// checkpoint
if (processContext.isInterrupted() && processContext.getProcessState() != ProcessState.MONITORING) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
List<TaskModel> taskList = processContext.getTaskList();
Map<String, TaskModel> taskMap = processContext.getTaskMap();
boolean fastForward = true;
for (String taskId : processContext.getTaskExecutionOrder()) {
if (fastForward) {
if (taskId.equalsIgnoreCase(startingTaskId)) {
fastForward = false;
} else {
continue;
}
}
TaskModel taskModel = taskMap.get(taskId);
processContext.setCurrentExecutingTaskModel(taskModel);
TaskTypes taskType = taskModel.getTaskType();
TaskContext taskContext = getTaskContext(processContext);
taskContext.setTaskModel(taskModel);
ProcessStatus status = null;
switch(taskType) {
case ENV_SETUP:
status = new ProcessStatus(ProcessState.CONFIGURING_WORKSPACE);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
// checkpoint
if (processContext.isInterrupted()) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
configureWorkspace(taskContext, processContext.isRecovery());
// checkpoint
if (processContext.isInterrupted()) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
break;
case DATA_STAGING:
try {
// checkpoint
if (processContext.isInterrupted()) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
DataStagingTaskModel subTaskModel = (DataStagingTaskModel) taskContext.getSubTaskModel();
DataStageType type = subTaskModel.getType();
switch(type) {
case INPUT:
status = new ProcessStatus(ProcessState.INPUT_DATA_STAGING);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
taskContext.setProcessInput(subTaskModel.getProcessInput());
inputDataStaging(taskContext, processContext.isRecovery());
break;
case OUPUT:
status = new ProcessStatus(ProcessState.OUTPUT_DATA_STAGING);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
taskContext.setProcessOutput(subTaskModel.getProcessOutput());
outputDataStaging(taskContext, processContext.isRecovery(), false);
break;
case ARCHIVE_OUTPUT:
status = new ProcessStatus(ProcessState.OUTPUT_DATA_STAGING);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
outputDataStaging(taskContext, processContext.isRecovery(), true);
break;
}
// checkpoint
if (processContext.isInterrupted()) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
} catch (TException e) {
throw new GFacException(e);
}
break;
case JOB_SUBMISSION:
// checkpoint
if (processContext.isInterrupted()) {
GFacUtils.handleProcessInterrupt(processContext);
return;
}
status = new ProcessStatus(ProcessState.EXECUTING);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
executeJobSubmission(taskContext, processContext.isRecovery());
// Don't put any checkpoint in between JobSubmission and Monitoring tasks
JobStatus jobStatus = processContext.getJobModel().getJobStatuses().get(0);
if (jobStatus != null && (jobStatus.getJobState() == JobState.SUBMITTED || jobStatus.getJobState() == JobState.QUEUED || jobStatus.getJobState() == JobState.ACTIVE)) {
List<OutputDataObjectType> processOutputs = processContext.getProcessModel().getProcessOutputs();
if (processOutputs != null && !processOutputs.isEmpty()) {
for (OutputDataObjectType output : processOutputs) {
try {
if (output.isOutputStreaming()) {
TaskModel streamingTaskModel = new TaskModel();
streamingTaskModel.setTaskType(TaskTypes.OUTPUT_FETCHING);
streamingTaskModel.setTaskStatuses(Arrays.asList(new TaskStatus(TaskState.CREATED)));
streamingTaskModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
streamingTaskModel.setParentProcessId(processContext.getProcessId());
TaskContext streamingTaskContext = getTaskContext(processContext);
DataStagingTaskModel submodel = new DataStagingTaskModel();
submodel.setType(DataStageType.OUPUT);
submodel.setProcessOutput(output);
URI source = new URI(processContext.getDataMovementProtocol().name(), processContext.getComputeResourceLoginUserName(), processContext.getComputeResourceDescription().getHostName(), 22, processContext.getWorkingDir() + output.getValue(), null, null);
submodel.setSource(source.getPath());
submodel.setDestination("dummy://temp/file/location");
streamingTaskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(submodel));
String streamTaskId = (String) processContext.getExperimentCatalog().add(ExpCatChildDataType.TASK, streamingTaskModel, processContext.getProcessId());
streamingTaskModel.setTaskId(streamTaskId);
streamingTaskContext.setTaskModel(streamingTaskModel);
executeDataStreaming(streamingTaskContext, processContext.isRecovery());
}
} catch (URISyntaxException | TException | RegistryException e) {
log.error("Error while streaming output " + output.getValue());
}
}
}
}
break;
case MONITORING:
status = new ProcessStatus(ProcessState.MONITORING);
status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
processContext.setProcessStatus(status);
GFacUtils.saveAndPublishProcessStatus(processContext);
executeJobMonitoring(taskContext, processContext.isRecovery());
break;
case ENV_CLEANUP:
// TODO implement environment clean up task logic
break;
default:
throw new GFacException("Unsupported Task type");
}
if (processContext.isPauseTaskExecution()) {
// If any task put processContext to wait, the same task must continue processContext execution.
return;
}
}
processContext.setComplete(true);
}
use of org.apache.airavata.model.status.JobStatus in project airavata by apache.
the class BESJobSubmissionTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
StorageClient sc = null;
ProcessContext processContext = taskContext.getParentProcessContext();
// FIXME - use original output dir
setInputOutputLocations(processContext);
try {
// con't reuse if UserDN has been changed.
secProperties = getSecurityConfig(processContext);
// try secProperties = secProperties.clone() if we can't use already initialized ClientConfigurations.
} catch (GFacException e) {
String msg = "Unicorn security context initialization error";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
return taskStatus;
}
try {
JobSubmissionProtocol protocol = processContext.getJobSubmissionProtocol();
JobSubmissionInterface jobSubmissionInterface = GFacUtils.getPreferredJobSubmissionInterface(processContext);
String factoryUrl = null;
if (protocol.equals(JobSubmissionProtocol.UNICORE)) {
UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(jobSubmissionInterface.getJobSubmissionInterfaceId());
factoryUrl = unicoreJobSubmission.getUnicoreEndPointURL();
}
EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
eprt.addNewAddress().setStringValue(factoryUrl);
String userDN = processContext.getProcessModel().getUserDn();
CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
// create storage
StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, null);
sc = storageCreator.createStorage();
JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(processContext, sc.getUrl()).getJobDefinition();
cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
log.info("Submitted JSDL: " + jobDefinition.getJobDescription());
// copy files to local
copyInputFilesToLocal(taskContext);
// upload files if any
DataTransferrer dt = new DataTransferrer(processContext, sc);
dt.uploadLocalFiles();
JobModel jobDetails = new JobModel();
jobDetails.setTaskId(taskContext.getTaskId());
jobDetails.setProcessId(taskContext.getProcessId());
FactoryClient factory = new FactoryClient(eprt, secProperties);
log.info("Activity Submitting to {} ... \n", factoryUrl);
CreateActivityResponseDocument response = factory.createActivity(cad);
log.info("Activity Submitted to {} ... \n", factoryUrl);
EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
log.info("Activity : " + activityEpr.getAddress().getStringValue() + " Submitted.");
// factory.waitWhileActivityIsDone(activityEpr, 1000);
jobId = WSUtilities.extractResourceID(activityEpr);
if (jobId == null) {
jobId = new Long(Calendar.getInstance().getTimeInMillis()).toString();
}
log.info("JobID: " + jobId);
jobDetails.setJobId(jobId);
jobDetails.setJobDescription(activityEpr.toString());
jobDetails.setJobStatuses(Arrays.asList(new JobStatus(JobState.SUBMITTED)));
processContext.setJobModel(jobDetails);
GFacUtils.saveJobModel(processContext, jobDetails);
GFacUtils.saveJobStatus(processContext, jobDetails);
log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), factory.getActivityStatus(activityEpr).toString()));
waitUntilDone(eprt, activityEpr, processContext, secProperties);
ActivityStatusType activityStatus = null;
activityStatus = getStatus(factory, activityEpr);
log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), activityStatus.getState().toString()));
ActivityClient activityClient;
activityClient = new ActivityClient(activityEpr, secProperties);
// now use the activity working directory property
dt.setStorageClient(activityClient.getUspaceClient());
List<OutputDataObjectType> copyOutput = null;
if ((activityStatus.getState() == ActivityStateEnumeration.FAILED)) {
String error = activityStatus.getFault().getFaultcode().getLocalPart() + "\n" + activityStatus.getFault().getFaultstring() + "\n EXITCODE: " + activityStatus.getExitCode();
log.error(error);
JobState applicationJobStatus = JobState.FAILED;
jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
sendNotification(processContext, jobDetails);
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
// What if job is failed before execution and there are not stdouts generated yet?
log.debug("Downloading any standard output and error files, if they were produced.");
copyOutput = dt.downloadRemoteFiles();
} else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
JobState applicationJobStatus = JobState.CANCELED;
jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
GFacUtils.saveJobStatus(processContext, jobDetails);
throw new GFacException(processContext.getExperimentId() + "Job Canceled");
} else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
try {
Thread.sleep(5000);
} catch (InterruptedException ignored) {
}
JobState applicationJobStatus = JobState.COMPLETE;
jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
GFacUtils.saveJobStatus(processContext, jobDetails);
log.info("Job Id: {}, exit code: {}, exit status: {}", jobDetails.getJobId(), activityStatus.getExitCode(), ActivityStateEnumeration.FINISHED.toString());
// if (activityStatus.getExitCode() == 0) {
// } else {
// dt.downloadStdOuts();
// }
copyOutput = dt.downloadRemoteFiles();
}
if (copyOutput != null) {
copyOutputFilesToStorage(taskContext, copyOutput);
for (OutputDataObjectType outputDataObjectType : copyOutput) {
GFacUtils.saveExperimentOutput(processContext, outputDataObjectType.getName(), outputDataObjectType.getValue());
}
}
// dt.publishFinalOutputs();
taskStatus.setState(TaskState.COMPLETED);
} catch (AppCatalogException e) {
log.error("Error while retrieving UNICORE job submission..", e);
taskStatus.setState(TaskState.FAILED);
} catch (Exception e) {
log.error("BES task failed... ", e);
taskStatus.setState(TaskState.FAILED);
}
return taskStatus;
}
use of org.apache.airavata.model.status.JobStatus in project airavata by apache.
the class ForkJobSubmissionTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
try {
ProcessContext processContext = taskContext.getParentProcessContext();
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
if (resourceJobManager != null) {
jConfig = Factory.getJobManagerConfiguration(resourceJobManager);
}
JobStatus jobStatus = new JobStatus();
File jobFile = GFacUtils.createJobFile(groovyMap, taskContext, jConfig);
if (jobFile != null && jobFile.exists()) {
jobModel.setJobDescription(FileUtils.readFileToString(jobFile));
JobSubmissionOutput jobSubmissionOutput = remoteCluster.submitBatchJob(jobFile.getPath(), processContext.getWorkingDir());
jobModel.setExitCode(jobSubmissionOutput.getExitCode());
jobModel.setStdErr(jobSubmissionOutput.getStdErr());
jobModel.setStdOut(jobSubmissionOutput.getStdOut());
String jobId = jobSubmissionOutput.getJobId();
if (jobId != null && !jobId.isEmpty()) {
jobModel.setJobId(jobId);
GFacUtils.saveJobModel(processContext, jobModel);
jobStatus.setJobState(JobState.SUBMITTED);
jobStatus.setReason("Successfully Submitted to " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
taskStatus = new TaskStatus(TaskState.COMPLETED);
taskStatus.setReason("Submitted job to compute resource");
}
if (jobId == null || jobId.isEmpty()) {
String msg = "expId:" + processContext.getProcessModel().getExperimentId() + " Couldn't find " + "remote jobId for JobName:" + jobModel.getJobName() + ", both submit and verify steps " + "doesn't return a valid JobId. " + "Hence changing experiment state to Failed";
log.error(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(msg);
errorModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
GFacUtils.saveExperimentError(processContext, errorModel);
GFacUtils.saveProcessError(processContext, errorModel);
GFacUtils.saveTaskError(taskContext, errorModel);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason("Couldn't find job id in both submitted and verified steps");
} else {
GFacUtils.saveJobModel(processContext, jobModel);
}
} else {
taskStatus.setState(TaskState.FAILED);
if (jobFile == null) {
taskStatus.setReason("JobFile is null");
} else {
taskStatus.setReason("Job file doesn't exist");
}
}
} catch (ApplicationSettingsException e) {
String msg = "Error occurred while creating job descriptor";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (AppCatalogException e) {
String msg = "Error while instantiating app catalog";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (GFacException e) {
String msg = "Error occurred while submitting the job";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (IOException e) {
String msg = "Error while reading the content of the job file";
log.error(msg, e);
taskStatus.setState(TaskState.FAILED);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
}
return taskStatus;
}
use of org.apache.airavata.model.status.JobStatus in project airavata by apache.
the class LocalJobSubmissionTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
try {
ProcessContext processContext = taskContext.getParentProcessContext();
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
String jobId = AiravataUtils.getId("JOB_ID_");
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
jobModel.setJobId(jobId);
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
if (resourceJobManager != null) {
jConfig = Factory.getJobManagerConfiguration(resourceJobManager);
}
JobStatus jobStatus = new JobStatus();
File jobFile = GFacUtils.createJobFile(groovyMap, taskContext, jConfig);
if (jobFile != null && jobFile.exists()) {
jobModel.setJobDescription(FileUtils.readFileToString(jobFile));
GFacUtils.saveJobModel(processContext, jobModel);
JobSubmissionOutput jobSubmissionOutput = remoteCluster.submitBatchJob(jobFile.getPath(), processContext.getWorkingDir());
jobStatus.setJobState(JobState.SUBMITTED);
jobStatus.setReason("Successfully Submitted to " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
// log job submit status
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
// for local, job gets completed synchronously
// so changing job status to complete
jobModel.setExitCode(jobSubmissionOutput.getExitCode());
jobModel.setStdErr(jobSubmissionOutput.getStdErr());
jobModel.setStdOut(jobSubmissionOutput.getStdOut());
jobModel.setJobId(jobId);
jobStatus.setJobState(JobState.COMPLETE);
jobStatus.setReason("Successfully Completed " + taskContext.getParentProcessContext().getComputeResourceDescription().getHostName());
jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
jobModel.setJobStatuses(Arrays.asList(jobStatus));
// log job complete status
GFacUtils.saveJobStatus(taskContext.getParentProcessContext(), jobModel);
taskStatus = new TaskStatus(TaskState.COMPLETED);
taskStatus.setReason("Submitted job to compute resource");
} else {
taskStatus.setState(TaskState.FAILED);
if (jobFile == null) {
taskStatus.setReason("JobFile is null");
} else {
taskStatus.setReason("Job file doesn't exist");
}
}
} catch (GFacException | IOException | AppCatalogException | ApplicationSettingsException e) {
String msg = "Error occurred while submitting a local job";
log.error(msg, e);
taskStatus.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
taskStatus.setState(TaskState.FAILED);
}
return taskStatus;
}
Aggregations