use of org.ow2.proactive.scheduler.common.job.factories.globalvariables.GlobalVariablesData in project scheduling by ow2-proactive.
the class StaxJobFactory method createAndFillJob.
/**
* Create the real job and fill it with its property. Leave the method at
* the first tag that define the real type of job.
*
* @param cursorJob the streamReader with the cursor on the job element.
* @param submittedJobVariables job submission variables map taking priority on those defined in the xml
* @param submittedGenericInfos job submission generic infos map taking priority on those defined in the xml
* @param jobContent contains xml representation of this job
* @throws JobCreationException if an exception occurs during job creation.
*/
private Job createAndFillJob(XMLStreamReader cursorJob, Map<String, String> submittedJobVariables, Map<String, String> submittedGenericInfos, String jobContent, GlobalVariablesData globalVariablesData) throws JobCreationException {
// A temporary job
Job commonPropertiesHolder = new Job() {
@Override
public JobId getId() {
throw new RuntimeException("Not Available!");
}
@Override
public JobType getType() {
throw new RuntimeException("Not Available!");
}
};
// To allow variable replacements on the job attributes, store them until job variables are parsed
Map<String, String> delayedJobAttributes = new LinkedHashMap<>();
int attrLen = cursorJob.getAttributeCount();
int i = 0;
for (; i < attrLen; i++) {
String attributeName = cursorJob.getAttributeLocalName(i);
String attributeValue = cursorJob.getAttributeValue(i);
delayedJobAttributes.put(attributeName, attributeValue);
}
Job job = commonPropertiesHolder;
try {
int eventType;
// Start by adding to the temporary job, the global variables (with internal references enabled)
commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(globalVariablesData.getVariables(), convertToReplacementMap(globalVariablesData.getVariables())));
commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(globalVariablesData.getGenericInformation(), commonPropertiesHolder.getVariablesAsReplacementMap()));
// Then add job submission variables, which will override eventually global variables
if (submittedJobVariables != null) {
for (String variableName : submittedJobVariables.keySet()) {
if (commonPropertiesHolder.getVariables().containsKey(variableName)) {
commonPropertiesHolder.getVariables().get(variableName).setValue(submittedJobVariables.get(variableName));
} else {
commonPropertiesHolder.getVariables().put(variableName, new JobVariable(variableName, submittedJobVariables.get(variableName), null));
}
}
// enable referencing of global variables by submitted variables
commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(commonPropertiesHolder.getVariables(), commonPropertiesHolder.getVariablesAsReplacementMap()));
}
// Then add job submission generic information, resolved using job submission and global variables
if (submittedGenericInfos != null) {
commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap()));
}
// Continue to fill the temporary job with xml elements
while (cursorJob.hasNext()) {
eventType = cursorJob.next();
if (eventType == XMLEvent.START_ELEMENT) {
String current = cursorJob.getLocalName();
if (XMLTags.VARIABLES.matches(current)) {
// Add resolved job variables using the job submission variables
// the final value of the variable can either be overwritten by a value of the job submission variables map or
// use in a pattern such value
Map<String, JobVariable> unresolvedJobVariablesMap = createUnresolvedJobVariables(cursorJob, globalVariablesData);
commonPropertiesHolder.getUnresolvedVariables().putAll(unresolvedJobVariablesMap);
Map<String, JobVariable> jobVariablesMap = replaceVariablesInJobVariablesMap(unresolvedJobVariablesMap, submittedJobVariables);
// this is to ensure preserving the order of variables defined in the workflow
for (String key : jobVariablesMap.keySet()) {
commonPropertiesHolder.getVariables().remove(key);
}
commonPropertiesHolder.getVariables().putAll(jobVariablesMap);
} else if (XMLTags.COMMON_GENERIC_INFORMATION.matches(current)) {
// Resolve the generic infos in the xml with the resolved variables
Map<String, String> resolvedJobVariables = commonPropertiesHolder.getVariablesAsReplacementMap();
Map<String, String> unresolvedGenericInformationsDefinedInWorkflow = getUnresolvedGenericInformations(cursorJob, false, globalVariablesData);
Map<String, String> resolvedGenericInformationsDefinedInWorkflow = getResolvedGenericInformations(unresolvedGenericInformationsDefinedInWorkflow, resolvedJobVariables);
// Then add/replace the resolved generic infos in the xml with the ones specified at job submission
if (submittedGenericInfos != null) {
Map<String, String> submittedGenericInformations = getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap());
resolvedGenericInformationsDefinedInWorkflow.putAll(submittedGenericInformations);
}
// Update the temporary job
commonPropertiesHolder.setGenericInformation(resolvedGenericInformationsDefinedInWorkflow);
commonPropertiesHolder.setUnresolvedGenericInformation(unresolvedGenericInformationsDefinedInWorkflow);
} else if (XMLTags.JOB_CLASSPATHES.matches(current)) {
logger.warn("Element " + XMLTags.JOB_CLASSPATHES.getXMLName() + " is no longer supported. Please define a " + XMLTags.FORK_ENVIRONMENT.getXMLName() + " per task if needed.");
} else if (XMLTags.COMMON_DESCRIPTION.matches(current)) {
commonPropertiesHolder.setDescription(getDescription(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
} else if (XMLTags.DS_INPUT_SPACE.matches(current)) {
commonPropertiesHolder.setInputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
} else if (XMLTags.DS_OUTPUT_SPACE.matches(current)) {
commonPropertiesHolder.setOutputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
} else if (XMLTags.DS_GLOBAL_SPACE.matches(current)) {
commonPropertiesHolder.setGlobalSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
} else if (XMLTags.DS_USER_SPACE.matches(current)) {
commonPropertiesHolder.setUserSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
} else if (XMLTags.TASK_FLOW.matches(current)) {
job = new TaskFlowJob();
// Stop cursor at the beginning of 'taskflow' tag, at this level all job properties are extracted except metadata
break;
}
}
}
handleJobAttributes(commonPropertiesHolder, delayedJobAttributes);
// if this point is reached, fill the real job using the temporary one
if (job != commonPropertiesHolder) {
job.setDescription(commonPropertiesHolder.getDescription());
job.setName(commonPropertiesHolder.getName());
job.setPriority(commonPropertiesHolder.getPriority());
job.setProjectName(commonPropertiesHolder.getProjectName());
job.setOnTaskError(commonPropertiesHolder.getOnTaskErrorProperty().getValue());
job.setRestartTaskOnError(commonPropertiesHolder.getRestartTaskOnError());
if (commonPropertiesHolder.getTaskRetryDelayProperty().isSet()) {
job.setTaskRetryDelay(commonPropertiesHolder.getTaskRetryDelay());
}
job.setMaxNumberOfExecution(commonPropertiesHolder.getMaxNumberOfExecution());
job.setGenericInformation(commonPropertiesHolder.getGenericInformation());
job.setUnresolvedGenericInformation(commonPropertiesHolder.getUnresolvedGenericInformation());
job.setInputSpace(commonPropertiesHolder.getInputSpace());
job.setOutputSpace(commonPropertiesHolder.getOutputSpace());
job.setGlobalSpace(commonPropertiesHolder.getGlobalSpace());
job.setUserSpace(commonPropertiesHolder.getUserSpace());
job.setVariables(commonPropertiesHolder.getVariables());
job.setUnresolvedVariables(commonPropertiesHolder.getUnresolvedVariables());
job.setVisualization(commonPropertiesHolder.getVisualization());
String updatedJobContent = getJobContentFactory.replaceVarsAndGenericInfo(jobContent, commonPropertiesHolder.getVariables(), commonPropertiesHolder.getGenericInformation());
job.setJobContent(updatedJobContent);
}
return job;
} catch (JobCreationException jce) {
jce.pushTag(cursorJob.getLocalName());
throw jce;
} catch (Exception e) {
String temporaryAttribute = null;
if (cursorJob.isStartElement() && cursorJob.getAttributeCount() > i) {
temporaryAttribute = cursorJob.getAttributeLocalName(i);
}
throw new JobCreationException(cursorJob.getLocalName(), temporaryAttribute, e);
}
}
use of org.ow2.proactive.scheduler.common.job.factories.globalvariables.GlobalVariablesData in project scheduling by ow2-proactive.
the class StaxJobFactory method createTask.
/**
* Fill the given task by the information that are at the given cursorTask.
* Leave the method with the cursor at the end of 'ELEMENT_TASK' tag.
*
* @param cursorTask the streamReader with the cursor on the 'ELEMENT_TASK' tag.
* @return The newly created task that can be any type.
*/
private Task createTask(XMLStreamReader cursorTask, Job job, Map<String, ArrayList<String>> dependencies, GlobalVariablesData globalVariablesData) throws JobCreationException {
int i = 0;
XMLTags currentTag = null;
String current = null;
String taskName = null;
try {
Task toReturn = null;
Task tmpTask = new Task() {
};
// To allow variable replacements on the task attributes, store them until task variables are parsed
Map<String, String> delayedTaskAttributes = new LinkedHashMap<>();
int attrLen = cursorTask.getAttributeCount();
for (; i < attrLen; i++) {
String attributeName = cursorTask.getAttributeLocalName(i);
String attributeValue = cursorTask.getAttributeValue(i);
delayedTaskAttributes.put(attributeName, attributeValue);
if (XMLAttributes.COMMON_NAME.matches(attributeName)) {
// it is currently not possible to use variables in task names (due to the complexity of such feature)
taskName = attributeValue;
tmpTask.setName(taskName);
}
}
// in the XML schema, description is defined after variables for tasks
String delayedDescription = null;
int eventType;
boolean shouldContinue = true;
while (shouldContinue && cursorTask.hasNext()) {
eventType = cursorTask.next();
switch(eventType) {
case XMLEvent.START_ELEMENT:
current = cursorTask.getLocalName();
currentTag = null;
if (XMLTags.COMMON_GENERIC_INFORMATION.matches(current)) {
// Resolve task generic infos using job variables and job generic infos
Map<String, String> jobVariablesWithGenericInfos = tmpTask.getVariablesOverriden(job);
if (job.getGenericInformation() != null)
jobVariablesWithGenericInfos.putAll(job.getGenericInformation());
Map<String, String> unresolvedGenericInformationDefinedInWorkflow = getUnresolvedGenericInformations(cursorTask, true, globalVariablesData);
tmpTask.setUnresolvedGenericInformation(unresolvedGenericInformationDefinedInWorkflow);
tmpTask.setGenericInformation(getResolvedGenericInformations(unresolvedGenericInformationDefinedInWorkflow, jobVariablesWithGenericInfos));
} else if (XMLTags.VARIABLES.matches(current)) {
// Add resolved task variables using both job and task variables
// the final value of the variable can either be overwritten by a value of the job submission variables map or
// use in a pattern such value
Map<String, TaskVariable> unresolvedTaskVariablesMap = createUnresolvedTaskVariables(cursorTask);
tmpTask.getUnresolvedVariables().putAll(unresolvedTaskVariablesMap);
Map<String, TaskVariable> taskVariablesMap = replaceVariablesInTaskVariablesMap(unresolvedTaskVariablesMap, getInitialTaskVariablesOverriden(job, unresolvedTaskVariablesMap));
tmpTask.setVariables(taskVariablesMap);
} else if (XMLTags.COMMON_DESCRIPTION.matches(current)) {
delayedDescription = getDescription(cursorTask, Collections.EMPTY_MAP);
} else if (XMLTags.DS_INPUT_FILES.matches(current)) {
setIOFIles(cursorTask, XMLTags.DS_INPUT_FILES.getXMLName(), tmpTask, tmpTask.getVariablesOverriden(job));
} else if (XMLTags.DS_OUTPUT_FILES.matches(current)) {
setIOFIles(cursorTask, XMLTags.DS_OUTPUT_FILES.getXMLName(), tmpTask, tmpTask.getVariablesOverriden(job));
} else if (XMLTags.PARALLEL_ENV.matches(current)) {
tmpTask.setParallelEnvironment(createParallelEnvironment(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.SCRIPT_SELECTION.matches(current)) {
tmpTask.setSelectionScripts(createSelectionScript(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.FORK_ENVIRONMENT.matches(current)) {
tmpTask.setForkEnvironment(createForkEnvironment(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.SCRIPT_PRE.matches(current)) {
tmpTask.setPreScript(createScript(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.SCRIPT_POST.matches(current)) {
tmpTask.setPostScript(createScript(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.SCRIPT_CLEANING.matches(current)) {
tmpTask.setCleaningScript(createScript(cursorTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.FLOW.matches(current)) {
tmpTask.setFlowScript(createControlFlowScript(cursorTask, tmpTask, tmpTask.getVariablesOverriden(job)));
} else if (XMLTags.TASK_DEPENDENCES.matches(current)) {
currentTag = XMLTags.TASK_DEPENDENCES;
dependencies.putAll(createDependences(cursorTask, tmpTask));
} else if (XMLTags.JAVA_EXECUTABLE.matches(current)) {
toReturn = new JavaTask();
setJavaExecutable((JavaTask) toReturn, cursorTask, tmpTask.getVariablesOverriden(job));
} else if (XMLTags.NATIVE_EXECUTABLE.matches(current)) {
toReturn = new NativeTask();
setNativeExecutable((NativeTask) toReturn, cursorTask);
} else if (XMLTags.SCRIPT_EXECUTABLE.matches(current)) {
toReturn = new ScriptTask();
((ScriptTask) toReturn).setScript(new TaskScript(createScript(cursorTask, tmpTask.getVariablesOverriden(job))));
}
break;
case XMLEvent.END_ELEMENT:
current = cursorTask.getLocalName();
if (XMLTags.TASK.matches(cursorTask.getLocalName())) {
shouldContinue = false;
}
break;
default:
}
}
// fill the task attributes using variable resolution
handleTaskAttributes(tmpTask, job, delayedTaskAttributes);
tmpTask.setDescription(replace(delayedDescription, tmpTask.getVariablesOverriden(job)));
// check whether task properties has conflicts between "runAsMe" and "fork"
if (tmpTask.isRunAsMe() && Boolean.FALSE.equals(tmpTask.isFork())) {
throw new JobCreationException(String.format("The task contains conflicting properties between 'runAsMe=%s' and 'fork=%s', because 'runAsMe=true' implies 'fork=true'.", tmpTask.isRunAsMe(), tmpTask.isFork()));
}
// fill the real task with common attribute if it is a new one
autoCopyfields(CommonAttribute.class, tmpTask, toReturn);
autoCopyfields(Task.class, tmpTask, toReturn);
if (toReturn != null) {
if (toReturn.getRestartTaskOnErrorProperty().isSet()) {
toReturn.setRestartTaskOnError(toReturn.getRestartTaskOnError());
}
if (toReturn.getMaxNumberOfExecutionProperty().isSet()) {
toReturn.setMaxNumberOfExecution(toReturn.getMaxNumberOfExecution());
}
}
return toReturn;
} catch (JobCreationException jce) {
jce.setTaskName(taskName);
if (currentTag != null) {
jce.pushTag(currentTag);
} else {
jce.pushTag(current);
}
throw jce;
} catch (Exception e) {
String attrtmp = null;
if (cursorTask.isStartElement() && cursorTask.getAttributeCount() > i) {
attrtmp = cursorTask.getAttributeLocalName(i);
}
if (currentTag != null) {
throw new JobCreationException(currentTag, attrtmp, e);
} else {
throw new JobCreationException(current, attrtmp, e);
}
}
}
use of org.ow2.proactive.scheduler.common.job.factories.globalvariables.GlobalVariablesData in project scheduling by ow2-proactive.
the class StaxJobFactory method createUnresolvedJobVariables.
/**
* Create a map of variables from XML variables.
* Leave the method with the cursor at the end of 'ELEMENT_VARIABLES' tag
*
* @param cursorVariables the streamReader with the cursor on the 'ELEMENT_VARIABLES' tag.
* @return the map in which the variables were added.
* @throws JobCreationException
*/
private Map<String, JobVariable> createUnresolvedJobVariables(XMLStreamReader cursorVariables, GlobalVariablesData globalVariablesData) throws JobCreationException {
// The following initializaion is to enable overridding of global variables by workflow variables
Map<String, JobVariable> unresolvedVariablesMap = globalVariablesData.getVariables();
try {
int eventType;
while (cursorVariables.hasNext()) {
eventType = cursorVariables.next();
switch(eventType) {
case XMLEvent.START_ELEMENT:
if (XMLTags.VARIABLE.matches(cursorVariables.getLocalName())) {
Map<String, String> unresolvedAttributesAsMap = getUnresolvedAttributesAsMap(cursorVariables);
String name = unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_NAME.getXMLName());
String value = unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_VALUE.getXMLName());
String model = unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_MODEL.getXMLName());
String description = unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_DESCRIPTION.getXMLName());
String group = unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_GROUP.getXMLName());
boolean advanced = false;
if (unresolvedAttributesAsMap.containsKey(XMLAttributes.VARIABLE_ADVANCED.getXMLName())) {
advanced = Boolean.parseBoolean(unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_ADVANCED.getXMLName()));
}
boolean hidden = false;
if (unresolvedAttributesAsMap.containsKey(XMLAttributes.VARIABLE_HIDDEN.getXMLName())) {
hidden = Boolean.parseBoolean(unresolvedAttributesAsMap.get(XMLAttributes.VARIABLE_HIDDEN.getXMLName()));
}
unresolvedVariablesMap.put(name, new JobVariable(name, value, model, description, group, advanced, hidden));
}
break;
case XMLEvent.END_ELEMENT:
if (XMLTags.VARIABLES.matches(cursorVariables.getLocalName())) {
return unresolvedVariablesMap;
}
break;
default:
}
}
} catch (JobCreationException jce) {
jce.pushTag(cursorVariables.getLocalName());
throw jce;
} catch (Exception e) {
String attrtmp = null;
if (cursorVariables.isStartElement() && cursorVariables.getAttributeCount() == 1) {
attrtmp = cursorVariables.getAttributeLocalName(0);
}
throw new JobCreationException(cursorVariables.getLocalName(), attrtmp, e);
}
return unresolvedVariablesMap;
}
use of org.ow2.proactive.scheduler.common.job.factories.globalvariables.GlobalVariablesData in project scheduling by ow2-proactive.
the class GlobalVariablesParser method getVariablesFor.
/**
* Return the global variables and generic information configured for the given workflow
* @param jobContent xml workflow as string
* @return global data containing variables and generic information
*/
public synchronized GlobalVariablesData getVariablesFor(String jobContent) {
GlobalVariablesData answer = new GlobalVariablesData();
Map<String, JobVariable> configuredVariables = new LinkedHashMap<>();
Map<String, String> configuredGenericInfo = new LinkedHashMap<>();
answer.setVariables(configuredVariables);
answer.setGenericInformation(configuredGenericInfo);
DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = null;
try {
builder = builderFactory.newDocumentBuilder();
} catch (ParserConfigurationException e) {
logger.error("Error when configuring DocumentBuilder", e);
return answer;
}
try (StringReader reader = new StringReader(jobContent)) {
InputSource inputSource = new InputSource(reader);
Document xmlDocument = builder.parse(inputSource);
XPath xPath = XPathFactory.newInstance().newXPath();
for (Filter filter : loadedFilters) {
boolean allMatch = true;
for (String xpathExpression : filter.getXpath()) {
NodeList nodeList = (NodeList) xPath.compile(xpathExpression).evaluate(xmlDocument, XPathConstants.NODESET);
allMatch = allMatch && (nodeList.getLength() > 0);
}
if (allMatch) {
for (JobVariable variable : filter.getVariables()) {
configuredVariables.put(variable.getName(), variable);
}
for (GenericInformation info : filter.getGenericInformation()) {
configuredGenericInfo.put(info.getName(), info.getValue());
}
}
}
} catch (Exception e) {
logger.error("Error analysing workflow global variables", e);
}
return answer;
}
use of org.ow2.proactive.scheduler.common.job.factories.globalvariables.GlobalVariablesData in project scheduling by ow2-proactive.
the class JobData method toInternalJob.
InternalJob toInternalJob() {
JobId jobIdInstance = new JobIdImpl(getId(), getJobName());
JobInfoImpl jobInfo = createJobInfo(jobIdInstance);
InternalJob internalJob = new InternalTaskFlowJob();
internalJob.setCredentials(getCredentials());
internalJob.setJobInfo(jobInfo);
internalJob.setParentId(getParentId());
internalJob.setGenericInformation(getGenericInformation());
internalJob.setVariables(variablesToJobVariables());
internalJob.setProjectName(getProjectName());
internalJob.setOwner(getOwner());
internalJob.setDescription(getDescription());
internalJob.setInputSpace(getInputSpace());
internalJob.setOutputSpace(getOutputSpace());
internalJob.setGlobalSpace(getGlobalSpace());
internalJob.setUserSpace(getGlobalSpace());
internalJob.setMaxNumberOfExecution(getMaxNumberOfExecution());
internalJob.setOnTaskError(OnTaskError.getInstance(this.onTaskErrorString));
if (getTaskRetryDelay() != null) {
internalJob.setTaskRetryDelay(getTaskRetryDelay());
}
internalJob.setScheduledTimeForRemoval(getScheduledTimeForRemoval());
try {
internalJob.setResultMap(SerializationUtil.deserializeVariableMap(getResultMap()));
} catch (IOException | ClassNotFoundException e) {
logger.error("error when serializing result map variables " + e);
}
List<JobContent> jobContentList = getJobContent();
if (jobContentList != null && jobContentList.size() > 0) {
internalJob.setJobContent(jobContentList.get(0).getInitJobContent());
if (internalJob.getJobContent() != null) {
GlobalVariablesData globalVariablesData = GlobalVariablesParser.getInstance().getVariablesFor(internalJob.getJobContent());
Map<String, JobVariable> globalVariables = new LinkedHashMap<>();
Map<String, JobVariable> configuredGlobalVariables = globalVariablesData.getVariables();
for (String variableName : configuredGlobalVariables.keySet()) {
if (internalJob.getVariables().containsKey(variableName)) {
globalVariables.put(variableName, internalJob.getVariables().get(variableName));
} else {
globalVariables.put(variableName, configuredGlobalVariables.get(variableName));
}
}
internalJob.setGlobalVariables(globalVariables);
Map<String, String> globalGenericInfo = new LinkedHashMap<>();
Map<String, String> configuredGlobalGenericInfo = globalVariablesData.getGenericInformation();
for (String giName : configuredGlobalGenericInfo.keySet()) {
if (internalJob.getGenericInformation().containsKey(giName)) {
globalGenericInfo.put(giName, internalJob.getGenericInformation().get(giName));
} else {
globalGenericInfo.put(giName, configuredGlobalGenericInfo.get(giName));
}
}
internalJob.setGlobalGenericInformation(globalGenericInfo);
}
}
return internalJob;
}
Aggregations