use of org.apache.airavata.registry.cpi.ExperimentCatalog in project airavata by apache.
the class SCPDataStageTask method execute.
@Override
public TaskStatus execute(TaskContext taskContext) {
TaskStatus status = new TaskStatus(TaskState.EXECUTING);
AuthenticationInfo authenticationInfo = null;
DataStagingTaskModel subTaskModel = null;
String localDataDir = null;
ProcessContext processContext = taskContext.getParentProcessContext();
ProcessState processState = processContext.getProcessState();
try {
subTaskModel = ((DataStagingTaskModel) taskContext.getSubTaskModel());
if (processState == ProcessState.OUTPUT_DATA_STAGING) {
OutputDataObjectType processOutput = taskContext.getProcessOutput();
if (processOutput != null && processOutput.getValue() == null) {
log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processOutput.getName());
status = new TaskStatus(TaskState.FAILED);
if (processOutput.isIsRequired()) {
status.setReason("File name is null, but this output's isRequired bit is not set");
} else {
status.setReason("File name is null");
}
return status;
}
} else if (processState == ProcessState.INPUT_DATA_STAGING) {
InputDataObjectType processInput = taskContext.getProcessInput();
if (processInput != null && processInput.getValue() == null) {
log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processInput.getName());
status = new TaskStatus(TaskState.FAILED);
if (processInput.isIsRequired()) {
status.setReason("File name is null, but this input's isRequired bit is not set");
} else {
status.setReason("File name is null");
}
return status;
}
} else {
status.setState(TaskState.FAILED);
status.setReason("Invalid task invocation, Support " + ProcessState.INPUT_DATA_STAGING.name() + " and " + "" + ProcessState.OUTPUT_DATA_STAGING.name() + " process phases. found " + processState.name());
return status;
}
StorageResourceDescription storageResource = processContext.getStorageResource();
// StoragePreference storagePreference = taskContext.getParentProcessContext().getStoragePreference();
String hostName = null;
if (storageResource != null) {
hostName = storageResource.getHostName();
} else {
throw new GFacException("Storage Resource is null");
}
String inputPath = processContext.getStorageFileSystemRootLocation();
inputPath = (inputPath.endsWith(File.separator) ? inputPath : inputPath + File.separator);
// use rsync instead of scp if source and destination host and user name is same.
URI sourceURI = new URI(subTaskModel.getSource());
String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1, sourceURI.getPath().length());
Session remoteSession = Factory.getSSHSession(Factory.getComputerResourceSSHKeyAuthentication(processContext), processContext.getComputeResourceServerInfo());
Session storageSession = Factory.getSSHSession(Factory.getStorageSSHKeyAuthentication(processContext), processContext.getStorageResourceServerInfo());
URI destinationURI = null;
if (subTaskModel.getDestination().startsWith("dummy")) {
destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath, fileName);
subTaskModel.setDestination(destinationURI.toString());
} else {
destinationURI = new URI(subTaskModel.getDestination());
}
if (sourceURI.getHost().equalsIgnoreCase(destinationURI.getHost()) && sourceURI.getUserInfo().equalsIgnoreCase(destinationURI.getUserInfo())) {
localDataCopy(taskContext, sourceURI, destinationURI);
status.setState(TaskState.COMPLETED);
status.setReason("Locally copied file using 'cp' command ");
return status;
}
status = new TaskStatus(TaskState.COMPLETED);
// Wildcard for file name. Has to find the correct name.
if (fileName.contains("*")) {
String destParentPath = (new File(destinationURI.getPath())).getParentFile().getPath();
String sourceParentPath = (new File(sourceURI.getPath())).getParentFile().getPath();
List<String> fileNames = taskContext.getParentProcessContext().getDataMovementRemoteCluster().getFileNameFromExtension(fileName, sourceParentPath, remoteSession);
ExperimentCatalog experimentCatalog = processContext.getExperimentCatalog();
String experimentId = processContext.getExperimentId();
String processId = processContext.getProcessId();
OutputDataObjectType processOutput = taskContext.getProcessOutput();
for (int i = 0; i < fileNames.size(); i++) {
String temp = fileNames.get(i);
if (temp != null && temp != "") {
fileName = temp;
}
if (destParentPath.endsWith(File.separator)) {
destinationURI = new URI(destParentPath + fileName);
} else {
destinationURI = new URI(destParentPath + File.separator + fileName);
}
// Wildcard support is only enabled for output data staging
if (processState == ProcessState.OUTPUT_DATA_STAGING) {
processOutput.setName(fileName);
experimentCatalog.add(ExpCatChildDataType.EXPERIMENT_OUTPUT, Arrays.asList(processOutput), experimentId);
experimentCatalog.add(ExpCatChildDataType.PROCESS_OUTPUT, Arrays.asList(processOutput), processId);
taskContext.setProcessOutput(processOutput);
makeDir(taskContext, destinationURI);
// TODO - save updated subtask model with new destination
outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
status.setReason("Successfully staged output data");
}
}
if (processState == ProcessState.OUTPUT_DATA_STAGING) {
status.setReason("Successfully staged output data");
} else {
status.setReason("Wildcard support is only enabled for output data staging");
}
} else {
if (processState == ProcessState.INPUT_DATA_STAGING) {
inputDataStaging(taskContext, storageSession, sourceURI, remoteSession, destinationURI);
status.setReason("Successfully staged input data");
} else if (processState == ProcessState.OUTPUT_DATA_STAGING) {
makeDir(taskContext, destinationURI);
// TODO - save updated subtask model with new destination
outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
status.setReason("Successfully staged output data");
}
}
} catch (TException e) {
String msg = "Couldn't create subTask model thrift model";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
return status;
} catch (ApplicationSettingsException | FileNotFoundException e) {
String msg = "Failed while reading credentials";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (URISyntaxException e) {
String msg = "Source or destination uri is not correct source : " + subTaskModel.getSource() + ", " + "destination : " + subTaskModel.getDestination();
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (CredentialStoreException e) {
String msg = "Storage authentication issue, could be invalid credential token";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (AiravataException e) {
String msg = "Error while creating ssh session with client";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (JSchException | IOException e) {
String msg = "Failed to do scp with client";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
} catch (RegistryException | GFacException e) {
String msg = "Data staging failed";
log.error(msg, e);
status.setState(TaskState.FAILED);
status.setReason(msg);
ErrorModel errorModel = new ErrorModel();
errorModel.setActualErrorMessage(e.getMessage());
errorModel.setUserFriendlyMessage(msg);
taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
}
return status;
}
use of org.apache.airavata.registry.cpi.ExperimentCatalog in project airavata by apache.
the class DataTransferrer method publishFinalOutputs.
public void publishFinalOutputs() throws GFacException {
try {
if (!resultantOutputsLst.isEmpty()) {
log.debug("Publishing the list of outputs to the registry instance..");
ExperimentCatalog experimentCatalog = processContext.getExperimentCatalog();
experimentCatalog.add(ExpCatChildDataType.EXPERIMENT_OUTPUT, resultantOutputsLst, processContext.getExperimentId());
}
} catch (RegistryException e) {
throw new GFacException("Cannot publish outputs to the registry.");
}
}
use of org.apache.airavata.registry.cpi.ExperimentCatalog in project airavata by apache.
the class WorkflowEngineImpl method launchExperiment.
@Override
public void launchExperiment(String experimentId, String token) throws WorkflowEngineException {
try {
ExperimentCatalog experimentCatalog = RegistryFactory.getDefaultExpCatalog();
Experiment experiment = (Experiment) experimentCatalog.get(ExperimentCatalogModelType.EXPERIMENT, experimentId);
WorkflowCatalog workflowCatalog = WorkflowCatalogFactory.getWorkflowCatalog();
WorkflowInterpreterConfiguration config = new WorkflowInterpreterConfiguration(new Workflow(workflowCatalog.getWorkflow(experiment.getApplicationId()).getGraph()));
final WorkflowInterpreter workflowInterpreter = new WorkflowInterpreter(experiment, token, config, getOrchestratorClient(), rabbitMQPublisher);
new Thread() {
public void run() {
try {
workflowInterpreter.scheduleDynamically();
} catch (WorkflowException e) {
logger.error(e.getMessage(), e);
} catch (RegistryException e) {
logger.error(e.getMessage(), e);
} catch (AiravataException e) {
logger.error(e.getMessage(), e);
}
}
}.start();
} catch (Exception e) {
logger.error("Error while retrieving the experiment", e);
WorkflowEngineException exception = new WorkflowEngineException("Error while launching the workflow experiment. More info : " + e.getMessage());
throw exception;
}
}
use of org.apache.airavata.registry.cpi.ExperimentCatalog in project airavata by apache.
the class GFacEngineImpl method populateProcessContext.
@Override
public ProcessContext populateProcessContext(String processId, String gatewayId, String tokenId) throws GFacException, CredentialStoreException {
// NOTE: Process context gives precedence to data come with process Computer resources;
ProcessContext processContext = null;
ProcessContext.ProcessContextBuilder builder = new ProcessContext.ProcessContextBuilder(processId, gatewayId, tokenId);
try {
AppCatalog appCatalog = Factory.getDefaultAppCatalog();
ExperimentCatalog expCatalog = Factory.getDefaultExpCatalog();
ProcessModel processModel = (ProcessModel) expCatalog.get(ExperimentCatalogModelType.PROCESS, processId);
builder.setAppCatalog(appCatalog).setExperimentCatalog(expCatalog).setCuratorClient(Factory.getCuratorClient()).setStatusPublisher(Factory.getStatusPublisher()).setProcessModel(processModel).setGatewayResourceProfile(appCatalog.getGatewayProfile().getGatewayProfile(gatewayId)).setGatewayComputeResourcePreference(appCatalog.getGatewayProfile().getComputeResourcePreference(gatewayId, processModel.getComputeResourceId())).setGatewayStorageResourcePreference(appCatalog.getGatewayProfile().getStoragePreference(gatewayId, processModel.getStorageResourceId()));
processContext = builder.build();
/* check point */
checkpoint(processContext);
if (processModel.isUseUserCRPref()) {
setUserResourceProfile(gatewayId, processContext);
setUserComputeResourcePreference(gatewayId, processContext);
}
String scratchLocation = processContext.getScratchLocation();
String workingDirectory = scratchLocation + File.separator + processId + File.separator;
StorageResourceDescription storageResource = appCatalog.getStorageResource().getStorageResource(processModel.getStorageResourceId());
if (storageResource != null) {
processContext.setStorageResource(storageResource);
} else {
// we need to fail the process which will fail the experiment
processContext.setProcessStatus(new ProcessStatus(ProcessState.FAILED));
GFacUtils.saveAndPublishProcessStatus(processContext);
throw new GFacException("expId: " + processModel.getExperimentId() + ", processId: " + processId + ":- Couldn't find storage resource for storage resource id :" + processModel.getStorageResourceId());
}
/* StorageResourceDescription storageResource = appCatalog.getStorageResource().getStorageResource(processModel.getStorageResourceId());
if (storageResource != null){
processContext.setStorageResource(storageResource);
}*/
processContext.setComputeResourceDescription(appCatalog.getComputeResource().getComputeResource(processContext.getComputeResourceId()));
processContext.setApplicationDeploymentDescription(appCatalog.getApplicationDeployment().getApplicationDeployement(processModel.getApplicationDeploymentId()));
ApplicationInterfaceDescription applicationInterface = appCatalog.getApplicationInterface().getApplicationInterface(processModel.getApplicationInterfaceId());
processContext.setApplicationInterfaceDescription(applicationInterface);
List<OutputDataObjectType> applicationOutputs = applicationInterface.getApplicationOutputs();
if (applicationOutputs != null && !applicationOutputs.isEmpty()) {
for (OutputDataObjectType outputDataObjectType : applicationOutputs) {
if (outputDataObjectType.getType().equals(DataType.STDOUT)) {
if (outputDataObjectType.getValue() == null || outputDataObjectType.getValue().equals("")) {
outputDataObjectType.setValue(workingDirectory + applicationInterface.getApplicationName() + ".stdout");
processContext.setStdoutLocation(workingDirectory + applicationInterface.getApplicationName() + ".stdout");
} else {
processContext.setStdoutLocation(outputDataObjectType.getValue());
}
}
if (outputDataObjectType.getType().equals(DataType.STDERR)) {
if (outputDataObjectType.getValue() == null || outputDataObjectType.getValue().equals("")) {
String stderrLocation = workingDirectory + applicationInterface.getApplicationName() + ".stderr";
outputDataObjectType.setValue(stderrLocation);
processContext.setStderrLocation(stderrLocation);
} else {
processContext.setStderrLocation(outputDataObjectType.getValue());
}
}
}
}
expCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processId);
processModel.setProcessOutputs(applicationOutputs);
if (processContext.getJobSubmissionProtocol() == JobSubmissionProtocol.UNICORE) {
// process monitor mode set in getResourceJobManager method, but unicore doesn't have resource job manager.
// hence we set process monitor mode here.
processContext.setMonitorMode(MonitorMode.FORK);
} else {
processContext.setResourceJobManager(getResourceJobManager(processContext));
processContext.setJobSubmissionRemoteCluster(Factory.getJobSubmissionRemoteCluster(processContext));
processContext.setDataMovementRemoteCluster(Factory.getDataMovementRemoteCluster(processContext));
}
String inputPath = ServerSettings.getLocalDataLocation();
if (inputPath != null) {
processContext.setLocalWorkingDir((inputPath.endsWith("/") ? inputPath : inputPath + "/") + processContext.getProcessId());
}
List<Object> jobModels = expCatalog.get(ExperimentCatalogModelType.JOB, "processId", processId);
if (jobModels != null && !jobModels.isEmpty()) {
if (jobModels.size() > 1) {
log.warn("Process has more than one job model, take first one");
}
processContext.setJobModel(((JobModel) jobModels.get(0)));
}
return processContext;
} catch (AppCatalogException e) {
String msg = "App catalog access exception ";
saveErrorModel(processContext, e, msg);
updateProcessFailure(processContext, msg);
throw new GFacException(msg, e);
} catch (RegistryException e) {
String msg = "Registry access exception";
saveErrorModel(processContext, e, msg);
updateProcessFailure(processContext, msg);
throw new GFacException(msg, e);
} catch (AiravataException e) {
String msg = "Remote cluster initialization error";
saveErrorModel(processContext, e, msg);
updateProcessFailure(processContext, msg);
throw new GFacException(msg, e);
}
}
Aggregations