Examples with SelectionScript org.ow2.proactive.scripting.SelectionScript used on opensource projects

Example 26 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class JobComparator method isTaskEqual.

private boolean isTaskEqual(Task t1, Task t2) throws IOException, ClassNotFoundException {
    if ((t1 == null) && (t2 == null))
        return true;
    if ((t1 == null) ^ (t2 == null)) {
        stack.push("One of 2 tasks is null");
        return false;
    }
    if (!isEqualCommonAttribute(t1, t2))
        return false;
    if (!t1.getName().equals(t2.getName())) {
        stack.push("name");
        return false;
    }
    if (!isEqualString(t1.getDescription(), t2.getDescription())) {
        stack.push("description");
        return false;
    }
    if (t1.getWallTime() != t2.getWallTime()) {
        stack.push("walltime");
        return false;
    }
    // ****** task dependencies ****
    stack.push("task dependenices");
    List<Task> dep1 = t1.getDependencesList();
    List<Task> dep2 = t2.getDependencesList();
    if (dep1 == null ^ dep2 == null) {
        stack.push("one dependency list is empty");
        return false;
    }
    if (dep1 != null) {
        if (dep1.size() != dep2.size()) {
            stack.push("sizes don't match");
            return false;
        }
        // we only compare the names in the 2 dependencies lists
        int dep1Size = dep1.size();
        Set<String> names1 = new HashSet<String>(dep1Size);
        Set<String> names2 = new HashSet<String>(dep1Size);
        for (int k = 0; k < dep1Size; k++) {
            names1.add(dep1.get(k).getName());
            names2.add(dep2.get(k).getName());
        }
        if (!CollectionUtils.isEqualCollection(names1, names2)) {
            return false;
        }
    }
    // task dependencies
    stack.pop();
    // **** parallel environment ****
    stack.push("parallel environment");
    if (!isEqualParallelEnvironment(t1.getParallelEnvironment(), t2.getParallelEnvironment()))
        return false;
    // parallel env
    stack.pop();
    // input files
    stack.push("input files");
    if (!isEqualInputFiles(t1.getInputFilesList(), t2.getInputFilesList()))
        return false;
    stack.pop();
    stack.push("output files");
    if (!isEqualOutputFiles(t1.getOutputFilesList(), t2.getOutputFilesList()))
        return false;
    stack.pop();
    // scripts
    stack.push("pre script");
    if (!isEqualScript(t1.getPreScript(), t2.getPreScript()))
        return false;
    stack.pop();
    stack.push("post script");
    if (!isEqualScript(t1.getPostScript(), t2.getPostScript()))
        return false;
    stack.pop();
    stack.push("cleaning script");
    if (!isEqualScript(t1.getCleaningScript(), t2.getCleaningScript()))
        return false;
    stack.pop();
    stack.push("selection scripts");
    List<SelectionScript> ss1 = t1.getSelectionScripts();
    List<SelectionScript> ss2 = t2.getSelectionScripts();
    if ((ss1 == null) ^ (ss2 == null)) {
        stack.push("One of two lists of selection scripts is null");
        return false;
    }
    if (ss1 != null) {
        if (t1.getSelectionScripts().size() != t2.getSelectionScripts().size()) {
            stack.push("lists size don't match");
            return false;
        }
        for (int k = 0; k < t1.getSelectionScripts().size(); k++) {
            if (!isEqualScript(t1.getSelectionScripts().get(k), t2.getSelectionScripts().get(k))) {
                return false;
            }
        }
    }
    // select scripts
    stack.pop();
    // flow control
    if (t1.getFlowBlock() != t2.getFlowBlock()) {
        stack.push("flow block");
        return false;
    }
    stack.push("flow control");
    if (!isEqualFlowControl(t1.getFlowScript(), t2.getFlowScript()))
        return false;
    stack.pop();
    // ***** task executable *****
    if (!isEqualClass(t1.getClass(), t2.getClass())) {
        stack.push("Executable types don't match");
        return false;
    }
    if (t1 instanceof JavaTask) {
        JavaTask jt1 = (JavaTask) t1;
        JavaTask jt2 = (JavaTask) t2;
        stack.push("arguments");
        if (!isEqualMap(jt1.getArguments(), jt2.getArguments()))
            return false;
        stack.pop();
        stack.push("executable class");
        if (!isEqualString(jt1.getExecutableClassName(), jt2.getExecutableClassName()))
            return false;
        stack.pop();
        stack.push("forked environemnt");
        if (!isEqualForkedEnvironment(jt1.getForkEnvironment(), jt2.getForkEnvironment()))
            return false;
        stack.pop();
    }
    if (t1 instanceof NativeTask) {
        NativeTask nt1 = (NativeTask) t1;
        NativeTask nt2 = (NativeTask) t2;
        String[] cl1 = nt1.getCommandLine();
        String[] cl2 = nt2.getCommandLine();
        if (cl1 == null ^ cl2 == null) {
            return false;
        } else if (cl1 != null) {
            if (!CollectionUtils.isEqualCollection(Arrays.asList(cl1), Arrays.asList(cl2))) {
                return false;
            }
        }
    }
    if (t1 instanceof ScriptTask) {
        ScriptTask st1 = (ScriptTask) t1;
        ScriptTask st2 = (ScriptTask) t2;
        if (!isEqualScript(st1.getScript(), st2.getScript()))
            return false;
    }
    return true;
}

Example 27 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class StaxJobFactory method createSelectionScript.

/**
 * Get the selection script defined at the specified cursor.
 * Leave the method with cursor at the end of the 'ELEMENT_SCRIPT_SELECTION' script.
 *
 * @param cursorScript the streamReader with the cursor on the 'ELEMENT_SCRIPT_SELECTION' tag.
 * @return the script defined at the specified cursor.
 */
private List<SelectionScript> createSelectionScript(XMLStreamReader cursorScript, Map<String, String> variables) throws JobCreationException {
    List<SelectionScript> scripts = new ArrayList<>(0);
    String selectionTag = cursorScript.getLocalName();
    String current = null;
    try {
        SelectionScript newOne;
        int eventType;
        while (cursorScript.hasNext()) {
            eventType = cursorScript.next();
            switch(eventType) {
                case XMLEvent.START_ELEMENT:
                    current = cursorScript.getLocalName();
                    if (XMLTags.SCRIPT_SCRIPT.matches(current)) {
                        newOne = (SelectionScript) createScript(cursorScript, ScriptType.SELECTION, variables);
                        scripts.add(newOne);
                    }
                    break;
                case XMLEvent.END_ELEMENT:
                    current = cursorScript.getLocalName();
                    if (current.equals(selectionTag)) {
                        if (scripts.size() == 0) {
                            return null;
                        } else {
                            return scripts;
                        }
                    }
                    break;
            }
        }
    } catch (JobCreationException jce) {
        jce.pushTag(current);
        throw jce;
    } catch (Exception e) {
        throw new JobCreationException(current, null, e);
    }
    return scripts;
}

Example 28 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class StaxJobFactory method createScript.

/**
 * Get the script defined at the specified cursor.
 * Leave the method with cursor at the end of the corresponding script.
 *
 * @param cursorScript the streamReader with the cursor on the corresponding script tag (pre, post, cleaning, selection, generation).
 * @param type         nature of the script
 * @return the script  defined at the specified cursor.
 */
private Script<?> createScript(XMLStreamReader cursorScript, ScriptType type, Map<String, String> variables) throws JobCreationException {
    String attrtmp = null;
    String currentScriptTag = cursorScript.getLocalName();
    String current = null;
    try {
        boolean isDynamic = true;
        Script<?> toReturn = null;
        int eventType = -1;
        while (cursorScript.hasNext()) {
            if (type == ScriptType.SELECTION && eventType == -1) {
                eventType = cursorScript.getEventType();
            } else {
                eventType = cursorScript.next();
            }
            switch(eventType) {
                case XMLEvent.START_ELEMENT:
                    current = cursorScript.getLocalName();
                    if (XMLTags.SCRIPT_CODE.matches(current)) {
                        String language = null;
                        String content = "";
                        if (cursorScript.getAttributeCount() > 0) {
                            language = cursorScript.getAttributeValue(0);
                            attrtmp = cursorScript.getAttributeLocalName(0);
                        }
                        // goto script content
                        if (cursorScript.next() == XMLEvent.CHARACTERS) {
                            content = cursorScript.getText();
                        }
                        toReturn = new SimpleScript(content, language);
                        // fast forward to the end of tag
                        while (true) {
                            int ev = cursorScript.next();
                            if (XMLTags.SCRIPT_CODE.matches(current) && ev == XMLEvent.END_ELEMENT) {
                                break;
                            }
                        }
                    } else if (XMLTags.SCRIPT_FILE.matches(current)) {
                        String path = null;
                        String url = null;
                        String language = null;
                        for (int i = 0; i < cursorScript.getAttributeCount(); i++) {
                            attrtmp = cursorScript.getAttributeLocalName(i);
                            if (XMLAttributes.SCRIPT_URL.matches(attrtmp)) {
                                url = replace(cursorScript.getAttributeValue(i), variables);
                            } else if (XMLAttributes.LANGUAGE.matches(attrtmp)) {
                                language = replace(cursorScript.getAttributeValue(i), variables);
                            } else if (XMLAttributes.PATH.matches(attrtmp)) {
                                path = checkPath(cursorScript.getAttributeValue(i), variables);
                            } else {
                                throw new JobCreationException("Unrecognized attribute : " + attrtmp);
                            }
                        }
                        // go to the next 'arguments' start element or the 'file' end element
                        while (true) {
                            int ev = cursorScript.next();
                            if (((ev == XMLEvent.START_ELEMENT) && XMLTags.SCRIPT_ARGUMENTS.matches(cursorScript.getLocalName())) || (ev == XMLEvent.END_ELEMENT)) {
                                break;
                            }
                        }
                        if (url != null) {
                            if (language != null) {
                                toReturn = new SimpleScript(new URL(url), language, getArguments(cursorScript));
                            } else {
                                toReturn = new SimpleScript(new URL(url), getArguments(cursorScript));
                            }
                        } else if (path != null) {
                            // language is ignored if a File is provided, the script language will be determined based on the file extension
                            toReturn = new SimpleScript(new File(path), getArguments(cursorScript));
                        } else {
                            attrtmp = null;
                            throw new JobCreationException("Invalid script file definition, one of path/url attributes must be declared");
                        }
                    } else if (XMLTags.SCRIPT_ARGUMENTS.matches(current)) {
                        toReturn = new SimpleScript(toReturn.getScript(), toReturn.getEngineName(), getArguments(cursorScript));
                    } else if (XMLTags.SCRIPT_SCRIPT.matches(current)) {
                        if (cursorScript.getAttributeCount() > 0) {
                            isDynamic = !"static".equals(cursorScript.getAttributeValue(0));
                        }
                    }
                    break;
                case XMLEvent.END_ELEMENT:
                    if (cursorScript.getLocalName().equals(currentScriptTag)) {
                        if (type == ScriptType.SELECTION) {
                            return new SelectionScript(toReturn, isDynamic);
                        } else {
                            return toReturn;
                        }
                    }
                    break;
            }
        }
        return toReturn;
    } catch (JobCreationException jce) {
        jce.pushTag(current);
        throw jce;
    } catch (Exception e) {
        throw new JobCreationException(current, attrtmp, e);
    }
}

Example 29 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class SelectionManager method doSelectNodes.

private NodeSet doSelectNodes(Criteria criteria, Client client) {
    boolean hasScripts = criteria.getScripts() != null && criteria.getScripts().size() > 0;
    boolean loggerIsDebugEnabled = logger.isDebugEnabled();
    if (loggerIsDebugEnabled) {
        logger.debug(client + " requested " + criteria.getSize() + " nodes with " + criteria.getTopology());
        if (hasScripts) {
            logger.debug("Selection scripts:");
            for (SelectionScript s : criteria.getScripts()) {
                logger.debug(s);
            }
        }
        if (criteria.getBlackList() != null && criteria.getBlackList().size() > 0) {
            logger.debug("Black list nodes:");
            for (Node n : criteria.getBlackList()) {
                logger.debug(n);
            }
        }
    }
    // can throw Exception if topology is disabled
    TopologyHandler handler = RMCore.topologyManager.getHandler(criteria.getTopology());
    int totalNumberOfAliveNodesRightNow = rmcore.getTotalAliveNodesNumber();
    List<RMNode> freeNodes = rmcore.getFreeNodes();
    // filtering out the "free node list"
    // removing exclusion and checking permissions
    List<RMNode> filteredNodes = filterOut(freeNodes, criteria, client);
    if (filteredNodes.size() == 0) {
        if (loggerIsDebugEnabled) {
            logger.debug(client + " will get 0 nodes");
        }
        return new NodeSet();
    }
    // arranging nodes according to the selection policy
    // if could be shuffling or node source priorities
    List<RMNode> afterPolicyNodes = selectionPolicy.arrangeNodes(criteria.getSize(), filteredNodes, client);
    List<Node> matchedNodes;
    if (hasScripts) {
        // checking if all scripts are authorized
        checkAuthorizedScripts(criteria.getScripts());
        // arranging nodes for script execution
        List<RMNode> arrangedNodes = arrangeNodesForScriptExecution(afterPolicyNodes, criteria.getScripts(), criteria.getBindings());
        List<RMNode> arrangedFilteredNodes = arrangedNodes;
        if (criteria.getTopology().isTopologyBased()) {
            arrangedFilteredNodes = topologyNodesFilter.filterNodes(criteria, arrangedNodes);
        }
        if (arrangedFilteredNodes.isEmpty()) {
            matchedNodes = new LinkedList<>();
        } else if (electedToRunOnAllNodes(criteria)) {
            // run scripts on all available nodes
            matchedNodes = runScripts(arrangedFilteredNodes, criteria);
        } else {
            // run scripts not on all nodes, but always on missing number of
            // nodes
            // until required node set is found
            matchedNodes = new LinkedList<>();
            while (matchedNodes.size() < criteria.getSize()) {
                int numberOfNodesForScriptExecution = criteria.getSize() - matchedNodes.size();
                if (numberOfNodesForScriptExecution < PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt()) {
                    // we can run
                    // "PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt()"
                    // scripts in parallel
                    // in case when we need less nodes it still useful to
                    // the full capacity of the thread pool to find nodes
                    // quicker
                    // it is not important if we find more nodes than needed
                    // subset will be selected later (topology handlers)
                    numberOfNodesForScriptExecution = PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt();
                }
                List<RMNode> subset = arrangedFilteredNodes.subList(0, Math.min(numberOfNodesForScriptExecution, arrangedFilteredNodes.size()));
                matchedNodes.addAll(runScripts(subset, criteria));
                // removing subset of arrangedNodes
                subset.clear();
                if (arrangedFilteredNodes.size() == 0) {
                    break;
                }
            }
            if (loggerIsDebugEnabled) {
                logger.debug(matchedNodes.size() + " nodes found after scripts execution for " + client);
            }
        }
    } else {
        matchedNodes = new LinkedList<>();
        for (RMNode node : afterPolicyNodes) {
            matchedNodes.add(node.getNode());
        }
    }
    if (criteria.getTopology().isTopologyBased() && loggerIsDebugEnabled) {
        logger.debug("Filtering nodes with topology " + criteria.getTopology());
    }
    NodeSet selectedNodes = handler.select(criteria.getSize(), matchedNodes);
    if (selectedNodes.size() < criteria.getSize() && !criteria.isBestEffort()) {
        selectedNodes.clear();
        if (selectedNodes.getExtraNodes() != null) {
            selectedNodes.getExtraNodes().clear();
        }
    }
    // the nodes are selected, now mark them as busy.
    for (Node node : selectedNodes) {
        try {
            // Synchronous call
            rmcore.setBusyNode(node.getNodeInformation().getURL(), client);
        } catch (NotConnectedException e) {
            // client has disconnected during getNodes request
            logger.warn(e.getMessage(), e);
            return null;
        }
    }
    // marking extra selected nodes as busy
    if (selectedNodes.size() > 0 && selectedNodes.getExtraNodes() != null) {
        for (Node node : new LinkedList<>(selectedNodes.getExtraNodes())) {
            try {
                // synchronous call
                rmcore.setBusyNode(node.getNodeInformation().getURL(), client);
            } catch (NotConnectedException e) {
                // client has disconnected during getNodes request
                logger.warn(e.getMessage(), e);
                return null;
            }
        }
    }
    if (logger.isInfoEnabled()) {
        String extraNodes = selectedNodes.getExtraNodes() != null && selectedNodes.getExtraNodes().size() > 0 ? " and " + selectedNodes.getExtraNodes().size() + " extra nodes" : "";
        logger.info(client + " requested " + criteria.getSize() + " nodes with " + criteria.getTopology() + " and will get " + selectedNodes.size() + " nodes " + extraNodes + " [totalNumberOfAliveNodesRightNow:" + totalNumberOfAliveNodesRightNow + ";freeNodes:" + freeNodes.size() + ";filteredNodes:" + filteredNodes.size() + ";reordered after policy:" + afterPolicyNodes.size() + ";selection script present:" + hasScripts + ";nodes filtered by selection script:" + matchedNodes.size() + ";selectedNodes:" + selectedNodes.size() + "]");
    }
    if (loggerIsDebugEnabled) {
        for (Node n : selectedNodes) {
            logger.debug(n.getNodeInformation().getURL());
        }
    }
    return selectedNodes;
}

Example 30 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class ProbablisticSelectionManager method processScriptResult.

/**
 * Processes script result and updates knowledge base of
 * selection manager at the same time.
 *
 * @param script - executed script
 * @param scriptResult - obtained script result
 * @param rmnode - node on which script has been executed
 * @return whether node is selected
 */
@Override
public synchronized boolean processScriptResult(SelectionScript script, Map<String, Serializable> bindings, ScriptResult<Boolean> scriptResult, RMNode rmnode) {
    boolean result = false;
    SelectionScript scriptWithReplacedBindings = replaceBindings(script, bindings);
    try {
        String digest = new String(scriptWithReplacedBindings.digest());
        Probability probability = new Probability(Probability.defaultValue());
        if (probabilities.containsKey(digest) && probabilities.get(digest).containsKey(rmnode.getNodeURL())) {
            probability = probabilities.get(digest).get(rmnode.getNodeURL());
            assert (probability.value() >= 0 && probability.value() <= 1);
        }
        if (scriptResult == null || scriptResult.errorOccured() || !scriptResult.getResult()) {
            // error during script execution or script returned false
            if (scriptWithReplacedBindings.isDynamic()) {
                probability.decrease();
            } else {
                probability = Probability.ZERO;
            }
        } else {
            // script passed
            result = true;
            if (scriptWithReplacedBindings.isDynamic()) {
                probability.increase();
            } else {
                probability = Probability.ONE;
            }
        }
        if (!probabilities.containsKey(digest)) {
            // checking if the number of selection script does not exceeded the maximum
            if (probabilities.size() >= PAResourceManagerProperties.RM_SELECT_SCRIPT_CACHE_SIZE.getValueAsInt()) {
                String oldest = digestQueue.poll();
                probabilities.remove(oldest);
                if (logger.isDebugEnabled()) {
                    logger.debug("Removing the script: " + scriptWithReplacedBindings.hashCode() + " from the data base because the limit is reached");
                }
            }
            // adding a new script record
            probabilities.put(digest, new HashMap<String, Probability>());
            logger.debug("Scripts cache size " + probabilities.size());
            digestQueue.offer(digest);
        }
        if (logger.isDebugEnabled()) {
            logger.debug(rmnode.getNodeURL() + " : script " + scriptWithReplacedBindings.hashCode() + ", probability " + probability);
        }
        probabilities.get(digest).put(rmnode.getNodeURL().intern(), probability);
    } catch (NoSuchAlgorithmException e) {
        logger.error(e.getMessage(), e);
    }
    return result;
}