Examples with WorkerSlot backtype.storm.scheduler.WorkerSlot used on opensource projects

Example 6 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class IsolatedPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    Set<Node> assignedNodes = new HashSet<Node>();
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            assignedNodes.add(n);
        }
    }
    _usedNodes += assignedNodes.size();
    _topologyIdToNodes.put(topId, assignedNodes);
    _tds.put(topId, td);
    if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) {
        _isolated.add(topId);
    }
}

Example 7 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class IsolatedPool method canAdd.

@Override
public boolean canAdd(TopologyDetails td) {
    // Only add topologies that are not sharing nodes with other topologies
    String topId = td.getId();
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            if (n.getRunningTopologies().size() > 1) {
                return false;
            }
        }
    }
    return true;
}

Example 8 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class Node method getAllNodesFrom.

public static Map<String, Node> getAllNodesFrom(Cluster cluster) {
    Map<String, Node> nodeIdToNode = new HashMap<String, Node>();
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        // Node ID and supervisor ID are the same.
        String id = sup.getId();
        boolean isAlive = !cluster.isBlackListed(id);
        LOG.debug("Found a {} Node {} {}", new Object[] { isAlive ? "living" : "dead", id, sup.getAllPorts() });
        nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
    }
    for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) {
        String topId = entry.getValue().getTopologyId();
        for (WorkerSlot ws : entry.getValue().getSlots()) {
            String id = ws.getNodeId();
            Node node = nodeIdToNode.get(id);
            if (node == null) {
                LOG.debug("Found an assigned slot on a dead supervisor {}", ws);
                node = new Node(id, null, false);
                nodeIdToNode.put(id, node);
            }
            if (!node.isAlive()) {
                // The supervisor on the node down so add an orphaned slot to hold the unsupervised worker
                node.addOrphanedSlot(ws);
            }
            if (node.assignInternal(ws, topId, true)) {
                LOG.warn("Bad scheduling state for topology [" + topId + "], the slot " + ws + " assigned to multiple workers, un-assigning everything...");
                node.free(ws, cluster, true);
            }
        }
    }
    return nodeIdToNode;
}

Example 9 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class Node method assign.

/**
     * Assign a free slot on the node to the following topology and executors. This will update the cluster too.
     * 
     * @param topId the topology to assign a free slot to.
     * @param executors the executors to run in that slot.
     * @param cluster the cluster to be updated
     */
public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) {
    if (!_isAlive) {
        throw new IllegalStateException("Trying to adding to a dead node " + _nodeId);
    }
    if (_freeSlots.isEmpty()) {
        throw new IllegalStateException("Trying to assign to a full node " + _nodeId);
    }
    if (executors.size() == 0) {
        LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)");
    } else {
        WorkerSlot slot = _freeSlots.iterator().next();
        cluster.assign(slot, topId, executors);
        assignInternal(slot, topId, false);
    }
}

Example 10 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class MkShuffer method isOutboundTaskAvailable.

private boolean isOutboundTaskAvailable(int taskId) {
    boolean ret = false;
    DisruptorQueue targetQueue = workerData.getInnerTaskTransfer().get(taskId);
    if (targetQueue != null) {
        float queueLoadRatio = targetQueue.pctFull();
        if (queueLoadRatio < loadMark) {
            ret = true;
        }
    } else {
        WorkerSlot slot = taskNodeport.get(taskId);
        if (slot != null) {
            IConnection connection = nodeportSocket.get(slot);
            if (connection != null) {
                ret = connection.available();
            }
        }
    }
    if (ret == false) {
        LOG.debug("taskId:{} is unavailable", taskId);
    }
    return ret;
}