Examples with NodeSet org.ow2.proactive.utils.NodeSet used on opensource projects

Example 31 with NodeSet

use of org.ow2.proactive.utils.NodeSet in project scheduling by ow2-proactive.

the class SelectionManager method doSelectNodes.

private NodeSet doSelectNodes(Criteria criteria, Client client) {
    boolean hasScripts = criteria.getScripts() != null && criteria.getScripts().size() > 0;
    boolean loggerIsDebugEnabled = logger.isDebugEnabled();
    if (loggerIsDebugEnabled) {
        logger.debug(client + " requested " + criteria.getSize() + " nodes with " + criteria.getTopology());
        if (hasScripts) {
            logger.debug("Selection scripts:");
            for (SelectionScript s : criteria.getScripts()) {
                logger.debug(s);
            }
        }
        if (criteria.getBlackList() != null && criteria.getBlackList().size() > 0) {
            logger.debug("Black list nodes:");
            for (Node n : criteria.getBlackList()) {
                logger.debug(n);
            }
        }
    }
    // can throw Exception if topology is disabled
    TopologyHandler handler = RMCore.topologyManager.getHandler(criteria.getTopology());
    int totalNumberOfAliveNodesRightNow = rmcore.getTotalAliveNodesNumber();
    List<RMNode> freeNodes = rmcore.getFreeNodes();
    // filtering out the "free node list"
    // removing exclusion and checking permissions
    List<RMNode> filteredNodes = filterOut(freeNodes, criteria, client);
    if (filteredNodes.size() == 0) {
        if (loggerIsDebugEnabled) {
            logger.debug(client + " will get 0 nodes");
        }
        return new NodeSet();
    }
    // arranging nodes according to the selection policy
    // if could be shuffling or node source priorities
    List<RMNode> afterPolicyNodes = selectionPolicy.arrangeNodes(criteria.getSize(), filteredNodes, client);
    List<Node> matchedNodes;
    if (hasScripts) {
        // checking if all scripts are authorized
        checkAuthorizedScripts(criteria.getScripts());
        // arranging nodes for script execution
        List<RMNode> arrangedNodes = arrangeNodesForScriptExecution(afterPolicyNodes, criteria.getScripts(), criteria.getBindings());
        List<RMNode> arrangedFilteredNodes = arrangedNodes;
        if (criteria.getTopology().isTopologyBased()) {
            arrangedFilteredNodes = topologyNodesFilter.filterNodes(criteria, arrangedNodes);
        }
        if (arrangedFilteredNodes.isEmpty()) {
            matchedNodes = new LinkedList<>();
        } else if (electedToRunOnAllNodes(criteria)) {
            // run scripts on all available nodes
            matchedNodes = runScripts(arrangedFilteredNodes, criteria);
        } else {
            // run scripts not on all nodes, but always on missing number of
            // nodes
            // until required node set is found
            matchedNodes = new LinkedList<>();
            while (matchedNodes.size() < criteria.getSize()) {
                int numberOfNodesForScriptExecution = criteria.getSize() - matchedNodes.size();
                if (numberOfNodesForScriptExecution < PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt()) {
                    // we can run
                    // "PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt()"
                    // scripts in parallel
                    // in case when we need less nodes it still useful to
                    // the full capacity of the thread pool to find nodes
                    // quicker
                    // it is not important if we find more nodes than needed
                    // subset will be selected later (topology handlers)
                    numberOfNodesForScriptExecution = PAResourceManagerProperties.RM_SELECTION_MAX_THREAD_NUMBER.getValueAsInt();
                }
                List<RMNode> subset = arrangedFilteredNodes.subList(0, Math.min(numberOfNodesForScriptExecution, arrangedFilteredNodes.size()));
                matchedNodes.addAll(runScripts(subset, criteria));
                // removing subset of arrangedNodes
                subset.clear();
                if (arrangedFilteredNodes.size() == 0) {
                    break;
                }
            }
            if (loggerIsDebugEnabled) {
                logger.debug(matchedNodes.size() + " nodes found after scripts execution for " + client);
            }
        }
    } else {
        matchedNodes = new LinkedList<>();
        for (RMNode node : afterPolicyNodes) {
            matchedNodes.add(node.getNode());
        }
    }
    if (criteria.getTopology().isTopologyBased() && loggerIsDebugEnabled) {
        logger.debug("Filtering nodes with topology " + criteria.getTopology());
    }
    NodeSet selectedNodes = handler.select(criteria.getSize(), matchedNodes);
    if (selectedNodes.size() < criteria.getSize() && !criteria.isBestEffort()) {
        selectedNodes.clear();
        if (selectedNodes.getExtraNodes() != null) {
            selectedNodes.getExtraNodes().clear();
        }
    }
    // the nodes are selected, now mark them as busy.
    for (Node node : selectedNodes) {
        try {
            // Synchronous call
            rmcore.setBusyNode(node.getNodeInformation().getURL(), client);
        } catch (NotConnectedException e) {
            // client has disconnected during getNodes request
            logger.warn(e.getMessage(), e);
            return null;
        }
    }
    // marking extra selected nodes as busy
    if (selectedNodes.size() > 0 && selectedNodes.getExtraNodes() != null) {
        for (Node node : new LinkedList<>(selectedNodes.getExtraNodes())) {
            try {
                // synchronous call
                rmcore.setBusyNode(node.getNodeInformation().getURL(), client);
            } catch (NotConnectedException e) {
                // client has disconnected during getNodes request
                logger.warn(e.getMessage(), e);
                return null;
            }
        }
    }
    if (logger.isInfoEnabled()) {
        String extraNodes = selectedNodes.getExtraNodes() != null && selectedNodes.getExtraNodes().size() > 0 ? " and " + selectedNodes.getExtraNodes().size() + " extra nodes" : "";
        logger.info(client + " requested " + criteria.getSize() + " nodes with " + criteria.getTopology() + " and will get " + selectedNodes.size() + " nodes " + extraNodes + " [totalNumberOfAliveNodesRightNow:" + totalNumberOfAliveNodesRightNow + ";freeNodes:" + freeNodes.size() + ";filteredNodes:" + filteredNodes.size() + ";reordered after policy:" + afterPolicyNodes.size() + ";selection script present:" + hasScripts + ";nodes filtered by selection script:" + matchedNodes.size() + ";selectedNodes:" + selectedNodes.size() + "]");
    }
    if (loggerIsDebugEnabled) {
        for (Node n : selectedNodes) {
            logger.debug(n.getNodeInformation().getURL());
        }
    }
    return selectedNodes;
}

Example 32 with NodeSet

use of org.ow2.proactive.utils.NodeSet in project scheduling by ow2-proactive.

the class SelectionManager method filterOut.

/**
 * Removes exclusion nodes and nodes not accessible for the client
 */
private List<RMNode> filterOut(List<RMNode> freeNodes, Criteria criteria, Client client) {
    NodeSet exclusion = criteria.getBlackList();
    Set<String> inclusion = criteria.getAcceptableNodesUrls();
    boolean nodeWithTokenRequested = criteria.getNodeAccessToken() != null && criteria.getNodeAccessToken().length() > 0;
    TokenPrincipal tokenPrincipal = null;
    if (nodeWithTokenRequested) {
        logger.debug("Node access token specified " + criteria.getNodeAccessToken());
        tokenPrincipal = new TokenPrincipal(criteria.getNodeAccessToken());
        client.getSubject().getPrincipals().add(tokenPrincipal);
    }
    List<RMNode> filteredList = new ArrayList<>();
    HashSet<Permission> clientPermissions = new HashSet<>();
    for (RMNode node : freeNodes) {
        // checking the permission
        try {
            if (!clientPermissions.contains(node.getUserPermission())) {
                client.checkPermission(node.getUserPermission(), client + " is not authorized to get the node " + node.getNodeURL() + " from " + node.getNodeSource().getName());
                clientPermissions.add(node.getUserPermission());
            }
        } catch (SecurityException e) {
            // client does not have an access to this node
            logger.debug(e.getMessage());
            continue;
        }
        // with other tokens but must also filter out nodes without tokens
        if (nodeWithTokenRequested && !node.isProtectedByToken()) {
            continue;
        }
        // we will avoid it here
        if (nodeWithTokenRequested) {
            PrincipalPermission perm = (PrincipalPermission) node.getUserPermission();
            // checking explicitly that node has this token identity
            if (!perm.hasPrincipal(tokenPrincipal)) {
                if (logger.isDebugEnabled()) {
                    logger.debug(client + " does not have required token to get the node " + node.getNodeURL() + " from " + node.getNodeSource().getName());
                }
                continue;
            }
        }
        if (!contains(exclusion, node) && ((inclusion != null) ? inclusion.contains(node.getNodeURL()) : true)) {
            filteredList.add(node);
        }
    }
    return filteredList;
}

Example 33 with NodeSet

use of org.ow2.proactive.utils.NodeSet in project scheduling by ow2-proactive.

the class ProbablisticSelectionManager method arrangeNodesForScriptExecution.

/**
 * Find appropriate candidates nodes for script execution, taking into
 * account "free" and "exclusion" nodes lists.
 *
 * @param scripts set of scripts to execute
 * @param nodes free nodes list provided by resource manager
 * @return candidates node list for script execution
 */
@Override
public List<RMNode> arrangeNodesForScriptExecution(final List<RMNode> nodes, List<SelectionScript> scripts, Map<String, Serializable> bindings) {
    long startTime = System.currentTimeMillis();
    boolean scriptSpecified = scripts != null && scripts.size() > 0;
    // if no scripts are specified return filtered free nodes
    if (!scriptSpecified) {
        return nodes;
    }
    try {
        // finding intersection
        HashMap<RMNode, Probability> intersectionMap = new LinkedHashMap<>();
        for (RMNode rmnode : nodes) {
            boolean intersection = true;
            double intersectionProbability = 1;
            for (SelectionScript script : scripts) {
                SelectionScript scriptWithReplacedBindings = replaceBindings(script, bindings);
                String digest = new String(scriptWithReplacedBindings.digest());
                if (probabilities.containsKey(digest) && probabilities.get(digest).containsKey(rmnode.getNodeURL())) {
                    double probability = probabilities.get(digest).get(rmnode.getNodeURL()).value();
                    if (probability == 0) {
                        intersection = false;
                        break;
                    } else {
                        intersectionProbability *= probability;
                    }
                } else {
                    intersectionProbability *= Probability.defaultValue();
                }
            }
            if (intersection) {
                intersectionMap.put(rmnode, new Probability(intersectionProbability));
            }
        }
        // sorting results based on calculated probability
        Set<RMNode> nodeSet = intersectionMap.keySet();
        List<RMNode> res = new ArrayList<>(nodeSet.size());
        res.addAll(nodeSet);
        Collections.sort(res, new NodeProbabilityComparator(intersectionMap));
        if (logger.isDebugEnabled()) {
            logger.debug("The following nodes are selected for scripts execution (time is " + (System.currentTimeMillis() - startTime) + " ms) :");
            if (res.size() > 0) {
                for (RMNode rmnode : res) {
                    logger.debug(rmnode.getNodeURL() + " : probability " + intersectionMap.get(rmnode));
                }
            } else {
                logger.debug("None");
            }
        }
        return res;
    } catch (NoSuchAlgorithmException e) {
        logger.error(e.getMessage(), e);
        return new ArrayList<>(0);
    }
}

Example 34 with NodeSet

use of org.ow2.proactive.utils.NodeSet in project scheduling by ow2-proactive.

the class TopologyManager method addNode.

/**
 * Updates the topology for new node. Executes the pinger on new node when this node belongs
 * to unknow host.
 */
public void addNode(Node node) {
    try {
        rwLock.writeLock().lock();
        if (!PAResourceManagerProperties.RM_TOPOLOGY_ENABLED.getValueAsBoolean()) {
            // do not do anything if topology disabled
            return;
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Adding Node " + node.getNodeInformation().getURL() + " to topology");
        }
        InetAddress host = node.getVMInformation().getInetAddress();
        if (topology.knownHost(host)) {
            // host topology is already known
            if (logger.isDebugEnabled()) {
                logger.debug("The topology information has been already added for node " + node.getNodeInformation().getURL());
            }
            nodesOnHost.get(host).add(node);
            return;
        }
        // unknown host => start pinging process
        NodeSet toPing = new NodeSet();
        HashMap<InetAddress, Long> hostsTopology = new HashMap<>();
        // adding one node from each host
        for (InetAddress h : nodesOnHost.keySet()) {
            // always have at least one node on each host
            if (nodesOnHost.get(h) != null && !nodesOnHost.get(h).isEmpty()) {
                toPing.add(nodesOnHost.get(h).iterator().next());
                hostsTopology.put(h, Long.MAX_VALUE);
            }
        }
        if (PAResourceManagerProperties.RM_TOPOLOGY_DISTANCE_ENABLED.getValueAsBoolean()) {
            hostsTopology = pingNode(node, toPing);
        }
        topology.addHostTopology(node.getVMInformation().getHostName(), host, hostsTopology);
        Set<Node> nodesList = new LinkedHashSet<>();
        nodesList.add(node);
        nodesOnHost.put(node.getVMInformation().getInetAddress(), nodesList);
    } finally {
        rwLock.writeLock().unlock();
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Node " + node.getNodeInformation().getURL() + " added.");
    }
}

Example 35 with NodeSet

use of org.ow2.proactive.utils.NodeSet in project scheduling by ow2-proactive.

the class TestRMMonitoring method action.

@Test
public void action() throws Exception {
    log("Deployment");
    RMMonitorEventReceiver resourceManager = (RMMonitorEventReceiver) rmHelper.getResourceManager();
    rmHelper.createNodeSource("TestRMMonitoring");
    Set<String> nodesUrls = rmHelper.listAliveNodesUrls();
    // we received all event as we are here
    log("Test 1 - subscribing only to 'node remove' event");
    resourceManager.removeRMEventListener();
    resourceManager.addRMEventListener(rmHelper.getEventReceiver(), RMEventType.NODE_REMOVED);
    String url = nodesUrls.iterator().next();
    BooleanWrapper status = resourceManager.removeNode(url, true);
    if (status.getBooleanValue()) {
        rmHelper.waitForAnyNodeEvent(RMEventType.NODE_REMOVED, 10000);
        log("Test 1 - success");
    }
    NodeSet ns = resourceManager.getAtMostNodes(5, null);
    log("Got " + ns.size() + " nodes");
    // must not receive "node busy" event
    try {
        rmHelper.waitForAnyNodeEvent(RMEventType.NODE_STATE_CHANGED, 2000);
        fail("Must not receive this type of event");
    } catch (ProActiveTimeoutException ex) {
        log("Test 2 - success");
    }
    resourceManager.releaseNodes(ns).getBooleanValue();
}