Examples with JobContent org.ow2.proactive.scheduler.core.db.JobContent used on opensource projects

Example 1 with JobContent

use of org.ow2.proactive.scheduler.core.db.JobContent in project scheduling by ow2-proactive.

the class TestWorkflowReSubmission method testAddVars.

@Test
public void testAddVars() throws Throwable {
    JobId jobId = schedulerHelper.submitJob(new File(jobDescriptor.toURI()).getAbsolutePath());
    Map<String, String> vars = new HashMap<>();
    vars.put("x", "50");
    JobId jobId1 = schedulerHelper.getSchedulerInterface().reSubmit(jobId, vars, null, null);
    schedulerHelper.waitForEventJobFinished(jobId);
    schedulerHelper.waitForEventJobFinished(jobId1);
    String jobContent = schedulerHelper.getSchedulerInterface().getJobContent(jobId);
    String jobContent1 = schedulerHelper.getSchedulerInterface().getJobContent(jobId1);
    assertFalse(jobContent.contains("<variables>"));
    assertFalse(jobContent.contains("<genericInformation>"));
    assertTrue(jobContent1.contains("<variables>"));
    assertFalse(jobContent1.contains("<genericInformation>"));
}

Example 2 with JobContent

use of org.ow2.proactive.scheduler.core.db.JobContent in project scheduling by ow2-proactive.

the class StaxJobFactory method createAndFillJob.

/**
 * Create the real job and fill it with its property. Leave the method at
 * the first tag that define the real type of job.
 *
 * @param cursorJob          the streamReader with the cursor on the job element.
 * @param submittedJobVariables job submission variables map taking priority on those defined in the xml
 * @param submittedGenericInfos job submission generic infos map taking priority on those defined in the xml
 * @param jobContent contains xml representation of this job
 * @throws JobCreationException if an exception occurs during job creation.
 */
private Job createAndFillJob(XMLStreamReader cursorJob, Map<String, String> submittedJobVariables, Map<String, String> submittedGenericInfos, String jobContent, GlobalVariablesData globalVariablesData) throws JobCreationException {
    // A temporary job
    Job commonPropertiesHolder = new Job() {

        @Override
        public JobId getId() {
            throw new RuntimeException("Not Available!");
        }

        @Override
        public JobType getType() {
            throw new RuntimeException("Not Available!");
        }
    };
    // To allow variable replacements on the job attributes, store them until job variables are parsed
    Map<String, String> delayedJobAttributes = new LinkedHashMap<>();
    int attrLen = cursorJob.getAttributeCount();
    int i = 0;
    for (; i < attrLen; i++) {
        String attributeName = cursorJob.getAttributeLocalName(i);
        String attributeValue = cursorJob.getAttributeValue(i);
        delayedJobAttributes.put(attributeName, attributeValue);
    }
    Job job = commonPropertiesHolder;
    try {
        int eventType;
        // Start by adding to the temporary job, the global variables (with internal references enabled)
        commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(globalVariablesData.getVariables(), convertToReplacementMap(globalVariablesData.getVariables())));
        commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(globalVariablesData.getGenericInformation(), commonPropertiesHolder.getVariablesAsReplacementMap()));
        // Then add job submission variables, which will override eventually global variables
        if (submittedJobVariables != null) {
            for (String variableName : submittedJobVariables.keySet()) {
                if (commonPropertiesHolder.getVariables().containsKey(variableName)) {
                    commonPropertiesHolder.getVariables().get(variableName).setValue(submittedJobVariables.get(variableName));
                } else {
                    commonPropertiesHolder.getVariables().put(variableName, new JobVariable(variableName, submittedJobVariables.get(variableName), null));
                }
            }
            // enable referencing of global variables by submitted variables
            commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(commonPropertiesHolder.getVariables(), commonPropertiesHolder.getVariablesAsReplacementMap()));
        }
        // Then add job submission generic information, resolved using job submission and global variables
        if (submittedGenericInfos != null) {
            commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap()));
        }
        // Continue to fill the temporary job with xml elements
        while (cursorJob.hasNext()) {
            eventType = cursorJob.next();
            if (eventType == XMLEvent.START_ELEMENT) {
                String current = cursorJob.getLocalName();
                if (XMLTags.VARIABLES.matches(current)) {
                    // Add resolved job variables using the job submission variables
                    // the final value of the variable can either be overwritten by a value of the job submission variables map or
                    // use in a pattern such value
                    Map<String, JobVariable> unresolvedJobVariablesMap = createUnresolvedJobVariables(cursorJob, globalVariablesData);
                    commonPropertiesHolder.getUnresolvedVariables().putAll(unresolvedJobVariablesMap);
                    Map<String, JobVariable> jobVariablesMap = replaceVariablesInJobVariablesMap(unresolvedJobVariablesMap, submittedJobVariables);
                    // this is to ensure preserving the order of variables defined in the workflow
                    for (String key : jobVariablesMap.keySet()) {
                        commonPropertiesHolder.getVariables().remove(key);
                    }
                    commonPropertiesHolder.getVariables().putAll(jobVariablesMap);
                } else if (XMLTags.COMMON_GENERIC_INFORMATION.matches(current)) {
                    // Resolve the generic infos in the xml with the resolved variables
                    Map<String, String> resolvedJobVariables = commonPropertiesHolder.getVariablesAsReplacementMap();
                    Map<String, String> unresolvedGenericInformationsDefinedInWorkflow = getUnresolvedGenericInformations(cursorJob, false, globalVariablesData);
                    Map<String, String> resolvedGenericInformationsDefinedInWorkflow = getResolvedGenericInformations(unresolvedGenericInformationsDefinedInWorkflow, resolvedJobVariables);
                    // Then add/replace the resolved generic infos in the xml with the ones specified at job submission
                    if (submittedGenericInfos != null) {
                        Map<String, String> submittedGenericInformations = getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap());
                        resolvedGenericInformationsDefinedInWorkflow.putAll(submittedGenericInformations);
                    }
                    // Update the temporary job
                    commonPropertiesHolder.setGenericInformation(resolvedGenericInformationsDefinedInWorkflow);
                    commonPropertiesHolder.setUnresolvedGenericInformation(unresolvedGenericInformationsDefinedInWorkflow);
                } else if (XMLTags.JOB_CLASSPATHES.matches(current)) {
                    logger.warn("Element " + XMLTags.JOB_CLASSPATHES.getXMLName() + " is no longer supported. Please define a " + XMLTags.FORK_ENVIRONMENT.getXMLName() + " per task if needed.");
                } else if (XMLTags.COMMON_DESCRIPTION.matches(current)) {
                    commonPropertiesHolder.setDescription(getDescription(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_INPUT_SPACE.matches(current)) {
                    commonPropertiesHolder.setInputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_OUTPUT_SPACE.matches(current)) {
                    commonPropertiesHolder.setOutputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_GLOBAL_SPACE.matches(current)) {
                    commonPropertiesHolder.setGlobalSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_USER_SPACE.matches(current)) {
                    commonPropertiesHolder.setUserSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.TASK_FLOW.matches(current)) {
                    job = new TaskFlowJob();
                    // Stop cursor at the beginning of 'taskflow' tag, at this level all job properties are extracted except metadata
                    break;
                }
            }
        }
        handleJobAttributes(commonPropertiesHolder, delayedJobAttributes);
        // if this point is reached, fill the real job using the temporary one
        if (job != commonPropertiesHolder) {
            job.setDescription(commonPropertiesHolder.getDescription());
            job.setName(commonPropertiesHolder.getName());
            job.setPriority(commonPropertiesHolder.getPriority());
            job.setProjectName(commonPropertiesHolder.getProjectName());
            job.setOnTaskError(commonPropertiesHolder.getOnTaskErrorProperty().getValue());
            job.setRestartTaskOnError(commonPropertiesHolder.getRestartTaskOnError());
            if (commonPropertiesHolder.getTaskRetryDelayProperty().isSet()) {
                job.setTaskRetryDelay(commonPropertiesHolder.getTaskRetryDelay());
            }
            job.setMaxNumberOfExecution(commonPropertiesHolder.getMaxNumberOfExecution());
            job.setGenericInformation(commonPropertiesHolder.getGenericInformation());
            job.setUnresolvedGenericInformation(commonPropertiesHolder.getUnresolvedGenericInformation());
            job.setInputSpace(commonPropertiesHolder.getInputSpace());
            job.setOutputSpace(commonPropertiesHolder.getOutputSpace());
            job.setGlobalSpace(commonPropertiesHolder.getGlobalSpace());
            job.setUserSpace(commonPropertiesHolder.getUserSpace());
            job.setVariables(commonPropertiesHolder.getVariables());
            job.setUnresolvedVariables(commonPropertiesHolder.getUnresolvedVariables());
            job.setVisualization(commonPropertiesHolder.getVisualization());
            String updatedJobContent = getJobContentFactory.replaceVarsAndGenericInfo(jobContent, commonPropertiesHolder.getVariables(), commonPropertiesHolder.getGenericInformation());
            job.setJobContent(updatedJobContent);
        }
        return job;
    } catch (JobCreationException jce) {
        jce.pushTag(cursorJob.getLocalName());
        throw jce;
    } catch (Exception e) {
        String temporaryAttribute = null;
        if (cursorJob.isStartElement() && cursorJob.getAttributeCount() > i) {
            temporaryAttribute = cursorJob.getAttributeLocalName(i);
        }
        throw new JobCreationException(cursorJob.getLocalName(), temporaryAttribute, e);
    }
}

Example 3 with JobContent

use of org.ow2.proactive.scheduler.core.db.JobContent in project scheduling by ow2-proactive.

the class GlobalVariablesParser method getVariablesFor.

/**
 * Return the global variables and generic information configured for the given workflow
 * @param jobContent xml workflow as string
 * @return global data containing variables and generic information
 */
public synchronized GlobalVariablesData getVariablesFor(String jobContent) {
    GlobalVariablesData answer = new GlobalVariablesData();
    Map<String, JobVariable> configuredVariables = new LinkedHashMap<>();
    Map<String, String> configuredGenericInfo = new LinkedHashMap<>();
    answer.setVariables(configuredVariables);
    answer.setGenericInformation(configuredGenericInfo);
    DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
    DocumentBuilder builder = null;
    try {
        builder = builderFactory.newDocumentBuilder();
    } catch (ParserConfigurationException e) {
        logger.error("Error when configuring DocumentBuilder", e);
        return answer;
    }
    try (StringReader reader = new StringReader(jobContent)) {
        InputSource inputSource = new InputSource(reader);
        Document xmlDocument = builder.parse(inputSource);
        XPath xPath = XPathFactory.newInstance().newXPath();
        for (Filter filter : loadedFilters) {
            boolean allMatch = true;
            for (String xpathExpression : filter.getXpath()) {
                NodeList nodeList = (NodeList) xPath.compile(xpathExpression).evaluate(xmlDocument, XPathConstants.NODESET);
                allMatch = allMatch && (nodeList.getLength() > 0);
            }
            if (allMatch) {
                for (JobVariable variable : filter.getVariables()) {
                    configuredVariables.put(variable.getName(), variable);
                }
                for (GenericInformation info : filter.getGenericInformation()) {
                    configuredGenericInfo.put(info.getName(), info.getValue());
                }
            }
        }
    } catch (Exception e) {
        logger.error("Error analysing workflow global variables", e);
    }
    return answer;
}

Example 4 with JobContent

use of org.ow2.proactive.scheduler.core.db.JobContent in project scheduling by ow2-proactive.

the class GetJobContentGeneratorTest method testReplaceVarsWithParams.

@Test
public void testReplaceVarsWithParams() throws IOException {
    URL url = Resources.getResource("org/ow2/proactive/scheduler/common/job/factories/job_with_vars_and_info.xml");
    String jobContent = Resources.toString(url, Charsets.UTF_8);
    Map<String, JobVariable> vars = new HashMap<>();
    vars.put("var", new JobVariable("var", "myvalue", "pa:model", "my var description", "varGroup", true, true));
    final String newJobContent = generator.replaceVarsAndGenericInfo(jobContent, vars, Collections.emptyMap());
    assertTrue(newJobContent.contains("<variables>"));
    assertTrue(newJobContent.contains("<variable name=\"var\" value=\"myvalue\" model=\"pa:model\" description=\"my var description\" group=\"varGroup\" advanced=\"true\" hidden=\"true\" />"));
    assertFalse(newJobContent.contains("<genericInformation>"));
    assertNotEquals(jobContent, newJobContent);
}

Example 5 with JobContent

use of org.ow2.proactive.scheduler.core.db.JobContent in project scheduling by ow2-proactive.

the class SchedulerFrontend method updateJobSchemaVersionToLatest.

/**
 * Use XSLT to change the schema version of the job descriptor to the latest.
 * Specifically, it's to change "xmlns" and "xsi:schemaLocation" to the "dev" version.
 * This function is used to fix the potential problem of schema version mismatch between global variables and job descriptor.
 * Since the schema change is backward compatible, updating the job to the latest version will fix the version mismatch problem.
 *
 * @param jobContent the String content of job descriptor (in xml)
 * @return the updated String content of job descriptor (in xml) which is changed to the latest version
 */
private String updateJobSchemaVersionToLatest(String jobContent) {
    try {
        StreamSource xslSource = new StreamSource(this.getClass().getClassLoader().getResourceAsStream(xslFilePath));
        StreamSource xmlInput = new StreamSource(new StringReader(jobContent));
        StringWriter xmlOutput = new StringWriter();
        Result result = new StreamResult(xmlOutput);
        TransformerFactory factory = TransformerFactory.newInstance(saxonFactoryClassName, null);
        Transformer transformer = factory.newTransformer(xslSource);
        transformer.transform(xmlInput, result);
        return xmlOutput.toString();
    } catch (Exception e) {
        logger.warn("Error during transforming the job descriptor schema to the latest version, it's kept unchanged.", e);
        return jobContent;
    }
}