Search in sources :

Example 6 with Script

use of org.ow2.proactive_grid_cloud_portal.studio.Script in project scheduling by ow2-proactive.

the class Attribute method createTaskElement.

/**
 * Creates the task element, corressponding to <define name="task">
 */
private Element createTaskElement(Document doc, Task task) {
    Element taskE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK.getXMLName());
    if (task.getOnTaskErrorProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_ON_TASK_ERROR, task.getOnTaskErrorProperty().getValue().toString(), true);
    }
    if (task.getMaxNumberOfExecutionProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION, Integer.toString(task.getMaxNumberOfExecution()));
    }
    setAttribute(taskE, XMLAttributes.COMMON_NAME, task.getName(), true);
    if (task.getRestartTaskOnErrorProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_RESTART_TASK_ON_ERROR, task.getRestartTaskOnError().toString());
    }
    // *** task attributes ***
    if (task.getWallTime() != 0) {
        setAttribute(taskE, XMLAttributes.TASK_WALLTIME, formatDate(task.getWallTime()));
    }
    if (task.isRunAsMe()) {
        setAttribute(taskE, XMLAttributes.TASK_RUN_AS_ME, "true");
    }
    if (task.isPreciousResult()) {
        setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_RESULT, "true");
    }
    if (task.isPreciousLogs()) {
        setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_LOGS, "true");
    }
    // <ref name="taskDescription"/>
    if (task.getDescription() != null) {
        Element descrNode = createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), task.getDescription());
        taskE.appendChild(descrNode);
    }
    // <ref name="variables"/>
    if (task.getVariables() != null && !task.getVariables().isEmpty()) {
        Element variablesE = createTaskVariablesElement(doc, task.getVariables());
        taskE.appendChild(variablesE);
    }
    // <ref name="genericInformation"/>
    if ((task.getGenericInformation() != null) && (task.getGenericInformation().size() > 0)) {
        Element genericInfoE = createGenericInformation(doc, task.getGenericInformation());
        taskE.appendChild(genericInfoE);
    }
    // <ref name="depends"/>
    List<Task> dependencies = task.getDependencesList();
    if ((dependencies != null) && (dependencies.size() > 0)) {
        Element dependsE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK_DEPENDENCES.getXMLName());
        for (Task dep : dependencies) {
            Element dependsTask = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK_DEPENDENCES_TASK.getXMLName());
            setAttribute(dependsTask, XMLAttributes.TASK_DEPENDS_REF, dep.getName(), true);
            dependsE.appendChild(dependsTask);
        }
        taskE.appendChild(dependsE);
    }
    // if has dependencies
    // <ref name="inputFiles"/>
    List<InputSelector> inputFiles = task.getInputFilesList();
    if (inputFiles != null) {
        Element inputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_INPUT_FILES.getXMLName());
        for (InputSelector inputSelector : inputFiles) {
            FileSelector fs = inputSelector.getInputFiles();
            Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_FILES.getXMLName());
            // pattern
            if (!fs.getIncludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
            if (!fs.getExcludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
            if (inputSelector.getMode() != null) {
                setAttribute(filesE, XMLAttributes.DS_ACCESS_MODE, inputSelector.getMode().toString(), true);
            }
            inputFilesE.appendChild(filesE);
        }
        taskE.appendChild(inputFilesE);
    }
    // <ref name="parallel"/>
    Element parallelEnvE = createParallelEnvironment(doc, task);
    if (parallelEnvE != null)
        taskE.appendChild(parallelEnvE);
    // <ref name="selection"/>
    List<SelectionScript> selectionScripts = task.getSelectionScripts();
    if (selectionScripts != null && selectionScripts.size() > 0) {
        Element selectionE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_SELECTION.getXMLName());
        for (SelectionScript selectionScript : selectionScripts) {
            Element scriptE = createScriptElement(doc, selectionScript);
            selectionE.appendChild(scriptE);
        }
        taskE.appendChild(selectionE);
    }
    // <ref name="forkEnvironment"/>
    if (task.getForkEnvironment() != null) {
        Element forkEnvE = createForkEnvironmentElement(doc, task.getForkEnvironment());
        taskE.appendChild(forkEnvE);
    }
    // <ref name="pre"/>
    Script preScript = task.getPreScript();
    if (preScript != null) {
        Element preE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_PRE.getXMLName());
        Element scriptE = createScriptElement(doc, preScript);
        preE.appendChild(scriptE);
        taskE.appendChild(preE);
    }
    // <ref name="executable"/>
    Element executableE = null;
    if (task instanceof JavaTask) {
        executableE = createJavaExecutableElement(doc, (JavaTask) task);
    } else if (task instanceof NativeTask) {
        executableE = createNativeExecutableElement(doc, (NativeTask) task);
    } else if (task instanceof ScriptTask) {
        executableE = createScriptExecutableElement(doc, (ScriptTask) task);
    }
    taskE.appendChild(executableE);
    // <ref name="flow"/>
    Element controlFlowE = createFlowControlElement(doc, task);
    if (controlFlowE != null)
        taskE.appendChild(controlFlowE);
    // <ref name="post"/>
    Script postScript = task.getPostScript();
    if (postScript != null) {
        Element postE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_POST.getXMLName());
        Element scriptE = createScriptElement(doc, postScript);
        postE.appendChild(scriptE);
        taskE.appendChild(postE);
    }
    // <ref name="cleaning"/>
    Script cleanScript = task.getCleaningScript();
    if (cleanScript != null) {
        Element cleanE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_CLEANING.getXMLName());
        Element scriptE = createScriptElement(doc, cleanScript);
        cleanE.appendChild(scriptE);
        taskE.appendChild(cleanE);
    }
    // <ref name="outputFiles"/>
    List<OutputSelector> outputFiles = task.getOutputFilesList();
    if (outputFiles != null) {
        Element outputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_OUTPUT_FILES.getXMLName());
        for (OutputSelector outputSelector : outputFiles) {
            FileSelector fs = outputSelector.getOutputFiles();
            Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_FILES.getXMLName());
            // pattern
            if (!fs.getIncludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
            if (!fs.getExcludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
            if (outputSelector.getMode() != null) {
                setAttribute(filesE, XMLAttributes.DS_ACCESS_MODE, outputSelector.getMode().toString(), true);
            }
            outputFilesE.appendChild(filesE);
        }
        taskE.appendChild(outputFilesE);
    }
    return taskE;
}
Also used : SelectionScript(org.ow2.proactive.scripting.SelectionScript) Script(org.ow2.proactive.scripting.Script) FlowScript(org.ow2.proactive.scheduler.common.task.flow.FlowScript) Task(org.ow2.proactive.scheduler.common.task.Task) JavaTask(org.ow2.proactive.scheduler.common.task.JavaTask) NativeTask(org.ow2.proactive.scheduler.common.task.NativeTask) ScriptTask(org.ow2.proactive.scheduler.common.task.ScriptTask) Element(org.w3c.dom.Element) FileSelector(org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector) JavaTask(org.ow2.proactive.scheduler.common.task.JavaTask) NativeTask(org.ow2.proactive.scheduler.common.task.NativeTask) SelectionScript(org.ow2.proactive.scripting.SelectionScript) ScriptTask(org.ow2.proactive.scheduler.common.task.ScriptTask) OutputSelector(org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector) InputSelector(org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector)

Example 7 with Script

use of org.ow2.proactive_grid_cloud_portal.studio.Script in project scheduling by ow2-proactive.

the class ScriptExecutor method executeScripts.

/**
 * Runs selection scripts and process the results
 * returns node if it matches, null otherwise
 */
private Node executeScripts() {
    boolean selectionScriptSpecified = selectionScriptList != null && selectionScriptList.size() > 0;
    boolean nodeMatch = true;
    ScriptException exception = null;
    if (selectionScriptSpecified) {
        // initializing parallel script execution
        for (SelectionScript script : selectionScriptList) {
            if (manager.isPassed(script, criteria.getBindings(), rmnode)) {
                // already executed static script
                logger.debug(rmnode.getNodeURL() + " : " + script.hashCode() + " skipping script execution");
                continue;
            }
            logger.info(rmnode.getNodeURL() + " : " + script.hashCode() + " executing");
            try {
                ScriptResult<Boolean> scriptResult = rmnode.executeScript(script, criteria.getBindings());
                // processing the results
                if (!MOP.isReifiedObject(scriptResult) && scriptResult.getException() != null) {
                    // could not create script execution handler
                    // probably the node id down
                    logger.warn(rmnode.getNodeURL() + " : " + script.hashCode() + " exception", scriptResult.getException());
                    logger.warn(rmnode.getNodeURL() + " : pinging the node");
                    rmnode.getNodeSource().pingNode(rmnode.getNode());
                    nodeMatch = false;
                    break;
                } else {
                    try {
                        PAFuture.waitFor(scriptResult, PAResourceManagerProperties.RM_SELECT_SCRIPT_TIMEOUT.getValueAsLong());
                    } catch (ProActiveTimeoutException e) {
                        logger.warn("Timeout on " + rmnode.getNodeURL());
                        // do not produce an exception here
                        nodeMatch = false;
                        break;
                    }
                    // display the script result and output in the scheduler logs
                    if (scriptResult != null && logger.isInfoEnabled()) {
                        logger.info(rmnode.getNodeURL() + " : " + script.hashCode() + " result " + scriptResult.getResult());
                        if (scriptResult.getOutput() != null && scriptResult.getOutput().length() > 0) {
                            logger.info(rmnode.getNodeURL() + " : " + script.hashCode() + " output\n" + scriptResult.getOutput());
                        }
                    }
                    if (scriptResult != null && scriptResult.errorOccured()) {
                        nodeMatch = false;
                        exception = new ScriptException(scriptResult.getException());
                        logger.warn(rmnode.getNodeURL() + " : exception during the script execution", scriptResult.getException());
                    }
                    // selection manager at the same time. Returns whether node is selected.
                    if (!manager.processScriptResult(script, criteria.getBindings(), scriptResult, rmnode)) {
                        nodeMatch = false;
                        break;
                    }
                }
            } catch (Exception ex) {
                // proactive or network exception occurred when script was executed
                logger.warn(rmnode.getNodeURL() + " : " + script.hashCode() + " exception", ex);
                nodeMatch = false;
                exception = new ScriptException(ex);
                break;
            }
        }
    }
    manager.scriptExecutionFinished(rmnode.getNodeURL());
    if (selectionScriptSpecified && logger.isDebugEnabled()) {
        if (nodeMatch) {
            logger.debug(rmnode.getNodeURL() + " : selected");
        } else {
            logger.debug(rmnode.getNodeURL() + " : not selected");
        }
    }
    // cleaning the node
    try {
        rmnode.clean();
    } catch (Throwable t) {
        logger.warn(rmnode.getNodeURL() + " : exception in cleaning", t);
        logger.warn(rmnode.getNodeURL() + " : pinging the node");
        try {
            // 'pingNode' call can fail with exception if NodeSource was destroyed
            rmnode.getNodeSource().pingNode(rmnode.getNode());
        } catch (Throwable pingError) {
            logger.warn(rmnode.getNodeURL() + " : nodeSource " + rmnode.getNodeSourceName() + " seems to be removed ", pingError);
        }
        return null;
    }
    if (exception != null) {
        throw exception;
    }
    if (nodeMatch) {
        return rmnode.getNode();
    } else {
        return null;
    }
}
Also used : ProActiveTimeoutException(org.objectweb.proactive.core.ProActiveTimeoutException) ScriptException(org.ow2.proactive.scripting.ScriptException) SelectionScript(org.ow2.proactive.scripting.SelectionScript) ScriptException(org.ow2.proactive.scripting.ScriptException) ProActiveTimeoutException(org.objectweb.proactive.core.ProActiveTimeoutException)

Example 8 with Script

use of org.ow2.proactive_grid_cloud_portal.studio.Script in project scheduling by ow2-proactive.

the class SelectionManager method executeScript.

public <T> List<ScriptResult<T>> executeScript(final Script<T> script, final Collection<RMNode> nodes, final Map<String, Serializable> bindings) {
    // TODO: add a specific timeout for script execution
    final long timeout = PAResourceManagerProperties.RM_EXECUTE_SCRIPT_TIMEOUT.getValueAsLong();
    final ArrayList<Callable<ScriptResult<T>>> scriptExecutors = new ArrayList<>(nodes.size());
    // Execute the script on each selected node
    for (final RMNode node : nodes) {
        scriptExecutors.add(new Callable<ScriptResult<T>>() {

            @Override
            public ScriptResult<T> call() throws Exception {
                // treated as ExecutionException
                try {
                    ScriptResult<T> res = node.executeScript(script, bindings);
                    PAFuture.waitFor(res, timeout);
                    return res;
                // return PAFuture.getFutureValue(res, timeout);
                } finally {
                    // cleaning the node
                    try {
                        node.clean();
                    } catch (Throwable ex) {
                        logger.error("Cannot clean the node " + node.getNodeURL(), ex);
                    }
                    SelectionManager.this.rmcore.unlockNodes(Collections.singleton(node.getNodeURL()));
                }
            }

            @Override
            public String toString() {
                return "executing script on " + node.getNodeURL();
            }
        });
    }
    // Invoke all Callables and get the list of futures
    List<Future<ScriptResult<T>>> futures = null;
    try {
        futures = this.scriptExecutorThreadPool.invokeAll(scriptExecutors, timeout, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        logger.warn("Interrupted while waiting, unable to execute all scripts", e);
        Thread.currentThread().interrupt();
    }
    final List<ScriptResult<T>> results = new LinkedList<>();
    int index = 0;
    // waiting for the results
    for (final Future<ScriptResult<T>> future : futures) {
        final String description = scriptExecutors.get(index++).toString();
        ScriptResult<T> result = null;
        try {
            result = future.get();
        } catch (CancellationException e) {
            result = new ScriptResult<>(new ScriptException("Cancelled due to timeout expiration when " + description, e));
        } catch (InterruptedException e) {
            result = new ScriptResult<>(new ScriptException("Cancelled due to interruption when " + description));
        } catch (ExecutionException e) {
            // Unwrap the root exception
            Throwable rex = e.getCause();
            result = new ScriptResult<>(new ScriptException("Exception occured in script call when " + description, rex));
        }
        results.add(result);
    }
    return results;
}
Also used : ArrayList(java.util.ArrayList) Callable(java.util.concurrent.Callable) ScriptException(org.ow2.proactive.scripting.ScriptException) NotConnectedException(org.ow2.proactive.resourcemanager.exception.NotConnectedException) CancellationException(java.util.concurrent.CancellationException) ExecutionException(java.util.concurrent.ExecutionException) LinkedList(java.util.LinkedList) ScriptResult(org.ow2.proactive.scripting.ScriptResult) ScriptException(org.ow2.proactive.scripting.ScriptException) RMNode(org.ow2.proactive.resourcemanager.rmnode.RMNode) CancellationException(java.util.concurrent.CancellationException) PAFuture(org.objectweb.proactive.api.PAFuture) Future(java.util.concurrent.Future) ExecutionException(java.util.concurrent.ExecutionException)

Example 9 with Script

use of org.ow2.proactive_grid_cloud_portal.studio.Script in project scheduling by ow2-proactive.

the class SelectionManager method loadAuthorizedScriptsSignatures.

private void loadAuthorizedScriptsSignatures(File folder) {
    authorizedSelectionScripts = new HashSet<>();
    for (File file : folder.listFiles()) {
        if (file.isFile()) {
            try {
                String script = SelectionScript.readFile(file);
                logger.debug("Adding authorized selection script " + file.getAbsolutePath());
                authorizedSelectionScripts.add(Script.digest(script.trim()));
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                throw new SecurityException("Error while reading authorized script file", e);
            }
        }
    }
}
Also used : File(java.io.File) ScriptException(org.ow2.proactive.scripting.ScriptException) NotConnectedException(org.ow2.proactive.resourcemanager.exception.NotConnectedException) CancellationException(java.util.concurrent.CancellationException) ExecutionException(java.util.concurrent.ExecutionException)

Example 10 with Script

use of org.ow2.proactive_grid_cloud_portal.studio.Script in project scheduling by ow2-proactive.

the class ProbablisticSelectionManager method isPassed.

/**
 * Predicts script execution result. Allows to avoid duplicate script execution
 * on the same node.
 *
 * @param script - script to execute
 * @param rmnode - target node
 * @return true if script will pass on the node
 */
@Override
public synchronized boolean isPassed(SelectionScript script, Map<String, Serializable> bindings, RMNode rmnode) {
    String digest;
    SelectionScript scriptWithReplacedBindings = replaceBindings(script, bindings);
    try {
        digest = new String(scriptWithReplacedBindings.digest());
        if (probabilities.containsKey(digest) && probabilities.get(digest).containsKey(rmnode.getNodeURL())) {
            Probability p = probabilities.get(digest).get(rmnode.getNodeURL());
            String scriptType = scriptWithReplacedBindings.isDynamic() ? "dynamic" : "static";
            if (logger.isDebugEnabled())
                logger.debug(rmnode.getNodeURL() + " : " + scriptWithReplacedBindings.hashCode() + " known " + scriptType + " script");
            return p.value() == 1;
        }
    } catch (NoSuchAlgorithmException e) {
        logger.error(e.getMessage(), e);
    }
    if (logger.isDebugEnabled())
        logger.debug(rmnode.getNodeURL() + " : " + scriptWithReplacedBindings.hashCode() + " unknown script");
    return false;
}
Also used : SelectionScript(org.ow2.proactive.scripting.SelectionScript) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException)

Aggregations

Test (org.junit.Test)38 SelectionScript (org.ow2.proactive.scripting.SelectionScript)32 File (java.io.File)31 SimpleScript (org.ow2.proactive.scripting.SimpleScript)28 TaskFlowJob (org.ow2.proactive.scheduler.common.job.TaskFlowJob)25 ArrayList (java.util.ArrayList)16 JavaTask (org.ow2.proactive.scheduler.common.task.JavaTask)15 Script (org.ow2.proactive.scripting.Script)15 ScriptResult (org.ow2.proactive.scripting.ScriptResult)15 FlowScript (org.ow2.proactive.scheduler.common.task.flow.FlowScript)13 HashMap (java.util.HashMap)12 RMNode (org.ow2.proactive.resourcemanager.rmnode.RMNode)11 IOException (java.io.IOException)10 JobId (org.ow2.proactive.scheduler.common.job.JobId)10 NodeSet (org.ow2.proactive.utils.NodeSet)10 RMFunctionalTest (functionaltests.utils.RMFunctionalTest)9 ResourceManager (org.ow2.proactive.resourcemanager.frontend.ResourceManager)9 Job (org.ow2.proactive.scheduler.common.job.Job)8 ForkEnvironment (org.ow2.proactive.scheduler.common.task.ForkEnvironment)8 Serializable (java.io.Serializable)7