Examples with JobVariable org.ow2.proactive.scheduler.common.job.JobVariable used on opensource projects

Example 6 with JobVariable

use of org.ow2.proactive.scheduler.common.job.JobVariable in project scheduling by ow2-proactive.

the class InProcessTaskExecutorTest method variablesPropagation.

@Test
public void variablesPropagation() throws Throwable {
    TestTaskOutput taskOutput = new TestTaskOutput();
    TaskLauncherInitializer initializer = new TaskLauncherInitializer();
    initializer.setPreScript(new SimpleScript("print(variables.get('var')); variables.put('var', 'pre')", "groovy"));
    initializer.setPostScript(new SimpleScript("print(variables.get('var')); variables.put('var', 'post')", "groovy"));
    initializer.setTaskId(TaskIdImpl.createTaskId(new JobIdImpl(1000, "job"), "task", 42L));
    initializer.setJobVariables(Collections.singletonMap("var", new JobVariable("var", "value")));
    TaskResultImpl result = new InProcessTaskExecutor().execute(new TaskContext(new ScriptExecutableContainer(new TaskScript(new SimpleScript("print(variables.get('var')); variables.put('var', 'task')", "groovy"))), initializer, null, new NodeDataSpacesURIs("", "", "", "", "", ""), "", new NodeInfo("", "", "", "")), taskOutput.outputStream, taskOutput.error);
    assertEquals("valuepretask", taskOutput.output());
    assertEquals("post", SerializationUtil.deserializeVariableMap(result.getPropagatedVariables()).get("var"));
}

Example 7 with JobVariable

use of org.ow2.proactive.scheduler.common.job.JobVariable in project scheduling by ow2-proactive.

the class TaskLauncherDataSpacesTest method input_file_using_variable_in_its_selector.

@Test
public void input_file_using_variable_in_its_selector() throws Throwable {
    ScriptExecutableContainer executableContainer = new ScriptExecutableContainer(new TaskScript(new SimpleScript("println new File('.').listFiles();", "groovy")));
    TaskLauncherInitializer initializer = new TaskLauncherInitializer();
    initializer.setTaskId(TaskIdImpl.createTaskId(JobIdImpl.makeJobId("1000"), "job", 1000L));
    initializer.setJobVariables(singletonMap("aVar", new JobVariable("aVar", "foo")));
    initializer.setTaskInputFiles(singletonList(new InputSelector(new FileSelector("input_${aVar}_${aResultVar}.txt"), InputAccessMode.TransferFromInputSpace)));
    File inputFile = new File(taskLauncherFactory.getDataSpaces().getInputURI(), "input_foo_bar.txt");
    assertTrue(inputFile.createNewFile());
    TaskResultImpl previousTaskResult = taskResult(Collections.<String, Serializable>singletonMap("aResultVar", "bar"));
    TaskResult taskResult = runTaskLauncher(createLauncherWithInjectedMocks(initializer, taskLauncherFactory), executableContainer, previousTaskResult);
    assertTaskResultOk(taskResult);
    assertTrue(taskResult.getOutput().getAllLogs(false).contains("input_foo_bar.txt"));
}

Example 8 with JobVariable

use of org.ow2.proactive.scheduler.common.job.JobVariable in project scheduling by ow2-proactive.

the class Attribute method createJobVariablesElement.

/*
     * Creates the job variables element
     */
private Element createJobVariablesElement(Document doc, Map<String, JobVariable> jobVariables) {
    if (jobVariables == null) {
        return null;
    }
    Element variablesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.VARIABLES.getXMLName());
    for (String name : jobVariables.keySet()) {
        JobVariable variable = jobVariables.get(name);
        List<Attribute> varAttributes = new ArrayList<>();
        varAttributes.add(new Attribute(XMLAttributes.VARIABLE_NAME.getXMLName(), name));
        varAttributes.add(new Attribute(XMLAttributes.VARIABLE_VALUE.getXMLName(), variable.getValue()));
        varAttributes.add(new Attribute(XMLAttributes.VARIABLE_MODEL.getXMLName(), variable.getModel()));
        if (variable.getDescription() != null) {
            varAttributes.add(new Attribute(XMLAttributes.VARIABLE_DESCRIPTION.getXMLName(), variable.getDescription()));
        }
        if (variable.getGroup() != null) {
            varAttributes.add(new Attribute(XMLAttributes.VARIABLE_GROUP.getXMLName(), variable.getGroup()));
        }
        if (variable.isAdvanced()) {
            varAttributes.add(new Attribute(XMLAttributes.VARIABLE_ADVANCED.getXMLName(), String.valueOf(variable.isAdvanced())));
        }
        if (variable.isHidden()) {
            varAttributes.add(new Attribute(XMLAttributes.VARIABLE_HIDDEN.getXMLName(), String.valueOf(variable.isHidden())));
        }
        Element variableE = createElement(doc, XMLTags.VARIABLE.getXMLName(), null, varAttributes.toArray(new Attribute[0]));
        variablesE.appendChild(variableE);
    }
    return variablesE;
}

Example 9 with JobVariable

use of org.ow2.proactive.scheduler.common.job.JobVariable in project scheduling by ow2-proactive.

the class StaxJobFactory method replaceVariablesInJobVariablesMap.

protected Map<String, JobVariable> replaceVariablesInJobVariablesMap(Map<String, JobVariable> variablesMap, Map<String, String> replacementVariables) throws JobCreationException {
    LinkedHashMap<String, String> updatedReplacementVariables = new LinkedHashMap<>();
    LinkedHashMap<String, JobVariable> updatedVariablesMap = new LinkedHashMap<>(variablesMap);
    // replacements will include at first variables defined in the job
    for (JobVariable variable : updatedVariablesMap.values()) {
        updatedReplacementVariables.put(variable.getName(), variable.getValue());
    }
    if (replacementVariables != null) {
        // overwritten by variables used at job submission
        updatedReplacementVariables.putAll(replacementVariables);
    }
    for (Map.Entry<String, String> replacementVariable : updatedReplacementVariables.entrySet()) {
        String replacementVariableKey = replacementVariable.getKey();
        if (updatedVariablesMap.containsKey(replacementVariableKey)) {
            // if the variable is already defined in the job, overwrite its value by the replacement variable,
            // eventually using other variables as pattern replacements
            JobVariable jobVariable = updatedVariablesMap.get(replacementVariableKey);
            JobVariable replacedJobVariable = new JobVariable(jobVariable.getName(), jobVariable.getValue(), jobVariable.getModel(), jobVariable.getDescription(), jobVariable.getGroup(), jobVariable.isAdvanced(), jobVariable.isHidden());
            replacedJobVariable.setValue(replace(replacementVariable.getValue(), updatedReplacementVariables));
            if (replacedJobVariable.getModel() != null) {
                // model of an existing variable can use other variables as pattern replacements
                replacedJobVariable.setModel(replace(replacedJobVariable.getModel(), updatedReplacementVariables));
            }
            updatedVariablesMap.put(replacementVariableKey, replacedJobVariable);
        } else {
            // if the variable is not defined in the job, create a new job variable with an empty model
            updatedVariablesMap.put(replacementVariableKey, new JobVariable(replacementVariableKey, replace(replacementVariable.getValue(), updatedReplacementVariables), null));
        }
    }
    return updatedVariablesMap;
}

Example 10 with JobVariable

use of org.ow2.proactive.scheduler.common.job.JobVariable in project scheduling by ow2-proactive.

the class StaxJobFactory method createAndFillJob.

/**
 * Create the real job and fill it with its property. Leave the method at
 * the first tag that define the real type of job.
 *
 * @param cursorJob          the streamReader with the cursor on the job element.
 * @param submittedJobVariables job submission variables map taking priority on those defined in the xml
 * @param submittedGenericInfos job submission generic infos map taking priority on those defined in the xml
 * @param jobContent contains xml representation of this job
 * @throws JobCreationException if an exception occurs during job creation.
 */
private Job createAndFillJob(XMLStreamReader cursorJob, Map<String, String> submittedJobVariables, Map<String, String> submittedGenericInfos, String jobContent, GlobalVariablesData globalVariablesData) throws JobCreationException {
    // A temporary job
    Job commonPropertiesHolder = new Job() {

        @Override
        public JobId getId() {
            throw new RuntimeException("Not Available!");
        }

        @Override
        public JobType getType() {
            throw new RuntimeException("Not Available!");
        }
    };
    // To allow variable replacements on the job attributes, store them until job variables are parsed
    Map<String, String> delayedJobAttributes = new LinkedHashMap<>();
    int attrLen = cursorJob.getAttributeCount();
    int i = 0;
    for (; i < attrLen; i++) {
        String attributeName = cursorJob.getAttributeLocalName(i);
        String attributeValue = cursorJob.getAttributeValue(i);
        delayedJobAttributes.put(attributeName, attributeValue);
    }
    Job job = commonPropertiesHolder;
    try {
        int eventType;
        // Start by adding to the temporary job, the global variables (with internal references enabled)
        commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(globalVariablesData.getVariables(), convertToReplacementMap(globalVariablesData.getVariables())));
        commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(globalVariablesData.getGenericInformation(), commonPropertiesHolder.getVariablesAsReplacementMap()));
        // Then add job submission variables, which will override eventually global variables
        if (submittedJobVariables != null) {
            for (String variableName : submittedJobVariables.keySet()) {
                if (commonPropertiesHolder.getVariables().containsKey(variableName)) {
                    commonPropertiesHolder.getVariables().get(variableName).setValue(submittedJobVariables.get(variableName));
                } else {
                    commonPropertiesHolder.getVariables().put(variableName, new JobVariable(variableName, submittedJobVariables.get(variableName), null));
                }
            }
            // enable referencing of global variables by submitted variables
            commonPropertiesHolder.getVariables().putAll(replaceVariablesInJobVariablesMap(commonPropertiesHolder.getVariables(), commonPropertiesHolder.getVariablesAsReplacementMap()));
        }
        // Then add job submission generic information, resolved using job submission and global variables
        if (submittedGenericInfos != null) {
            commonPropertiesHolder.addGenericInformations(getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap()));
        }
        // Continue to fill the temporary job with xml elements
        while (cursorJob.hasNext()) {
            eventType = cursorJob.next();
            if (eventType == XMLEvent.START_ELEMENT) {
                String current = cursorJob.getLocalName();
                if (XMLTags.VARIABLES.matches(current)) {
                    // Add resolved job variables using the job submission variables
                    // the final value of the variable can either be overwritten by a value of the job submission variables map or
                    // use in a pattern such value
                    Map<String, JobVariable> unresolvedJobVariablesMap = createUnresolvedJobVariables(cursorJob, globalVariablesData);
                    commonPropertiesHolder.getUnresolvedVariables().putAll(unresolvedJobVariablesMap);
                    Map<String, JobVariable> jobVariablesMap = replaceVariablesInJobVariablesMap(unresolvedJobVariablesMap, submittedJobVariables);
                    // this is to ensure preserving the order of variables defined in the workflow
                    for (String key : jobVariablesMap.keySet()) {
                        commonPropertiesHolder.getVariables().remove(key);
                    }
                    commonPropertiesHolder.getVariables().putAll(jobVariablesMap);
                } else if (XMLTags.COMMON_GENERIC_INFORMATION.matches(current)) {
                    // Resolve the generic infos in the xml with the resolved variables
                    Map<String, String> resolvedJobVariables = commonPropertiesHolder.getVariablesAsReplacementMap();
                    Map<String, String> unresolvedGenericInformationsDefinedInWorkflow = getUnresolvedGenericInformations(cursorJob, false, globalVariablesData);
                    Map<String, String> resolvedGenericInformationsDefinedInWorkflow = getResolvedGenericInformations(unresolvedGenericInformationsDefinedInWorkflow, resolvedJobVariables);
                    // Then add/replace the resolved generic infos in the xml with the ones specified at job submission
                    if (submittedGenericInfos != null) {
                        Map<String, String> submittedGenericInformations = getResolvedGenericInformations(submittedGenericInfos, commonPropertiesHolder.getVariablesAsReplacementMap());
                        resolvedGenericInformationsDefinedInWorkflow.putAll(submittedGenericInformations);
                    }
                    // Update the temporary job
                    commonPropertiesHolder.setGenericInformation(resolvedGenericInformationsDefinedInWorkflow);
                    commonPropertiesHolder.setUnresolvedGenericInformation(unresolvedGenericInformationsDefinedInWorkflow);
                } else if (XMLTags.JOB_CLASSPATHES.matches(current)) {
                    logger.warn("Element " + XMLTags.JOB_CLASSPATHES.getXMLName() + " is no longer supported. Please define a " + XMLTags.FORK_ENVIRONMENT.getXMLName() + " per task if needed.");
                } else if (XMLTags.COMMON_DESCRIPTION.matches(current)) {
                    commonPropertiesHolder.setDescription(getDescription(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_INPUT_SPACE.matches(current)) {
                    commonPropertiesHolder.setInputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_OUTPUT_SPACE.matches(current)) {
                    commonPropertiesHolder.setOutputSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_GLOBAL_SPACE.matches(current)) {
                    commonPropertiesHolder.setGlobalSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.DS_USER_SPACE.matches(current)) {
                    commonPropertiesHolder.setUserSpace(getIOSpace(cursorJob, commonPropertiesHolder.getVariablesAsReplacementMap()));
                } else if (XMLTags.TASK_FLOW.matches(current)) {
                    job = new TaskFlowJob();
                    // Stop cursor at the beginning of 'taskflow' tag, at this level all job properties are extracted except metadata
                    break;
                }
            }
        }
        handleJobAttributes(commonPropertiesHolder, delayedJobAttributes);
        // if this point is reached, fill the real job using the temporary one
        if (job != commonPropertiesHolder) {
            job.setDescription(commonPropertiesHolder.getDescription());
            job.setName(commonPropertiesHolder.getName());
            job.setPriority(commonPropertiesHolder.getPriority());
            job.setProjectName(commonPropertiesHolder.getProjectName());
            job.setOnTaskError(commonPropertiesHolder.getOnTaskErrorProperty().getValue());
            job.setRestartTaskOnError(commonPropertiesHolder.getRestartTaskOnError());
            if (commonPropertiesHolder.getTaskRetryDelayProperty().isSet()) {
                job.setTaskRetryDelay(commonPropertiesHolder.getTaskRetryDelay());
            }
            job.setMaxNumberOfExecution(commonPropertiesHolder.getMaxNumberOfExecution());
            job.setGenericInformation(commonPropertiesHolder.getGenericInformation());
            job.setUnresolvedGenericInformation(commonPropertiesHolder.getUnresolvedGenericInformation());
            job.setInputSpace(commonPropertiesHolder.getInputSpace());
            job.setOutputSpace(commonPropertiesHolder.getOutputSpace());
            job.setGlobalSpace(commonPropertiesHolder.getGlobalSpace());
            job.setUserSpace(commonPropertiesHolder.getUserSpace());
            job.setVariables(commonPropertiesHolder.getVariables());
            job.setUnresolvedVariables(commonPropertiesHolder.getUnresolvedVariables());
            job.setVisualization(commonPropertiesHolder.getVisualization());
            String updatedJobContent = getJobContentFactory.replaceVarsAndGenericInfo(jobContent, commonPropertiesHolder.getVariables(), commonPropertiesHolder.getGenericInformation());
            job.setJobContent(updatedJobContent);
        }
        return job;
    } catch (JobCreationException jce) {
        jce.pushTag(cursorJob.getLocalName());
        throw jce;
    } catch (Exception e) {
        String temporaryAttribute = null;
        if (cursorJob.isStartElement() && cursorJob.getAttributeCount() > i) {
            temporaryAttribute = cursorJob.getAttributeLocalName(i);
        }
        throw new JobCreationException(cursorJob.getLocalName(), temporaryAttribute, e);
    }
}