Examples with SupervisorInfo com.alibaba.jstorm.daemon.supervisor.SupervisorInfo used on opensource projects

Example 6 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class ZooKeeperDataViewTest method viewSupervisors.

@Test
public void viewSupervisors() throws Exception {
    if (SKIP == true) {
        return;
    }
    List<String> assignments = zkobj.getChildren(zk, Cluster.SUPERVISORS_SUBTREE, false);
    for (String child : assignments) {
        byte[] data = zkobj.getData(zk, Cluster.supervisor_path(child), false);
        SupervisorInfo supervisorInfo = (SupervisorInfo) Utils.maybe_deserialize(data);
        System.out.println(gson.toJson(supervisorInfo));
    }
}

Example 7 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class DefaultTopologyAssignContext method generateSidToHost.

/**
     * @@@ Do we need just handle the case whose type is ASSIGN_TYPE_NEW?
     * 
     * @return
     */
private Map<String, String> generateSidToHost() {
    Map<String, String> sidToHostname = new HashMap<String, String>();
    if (oldAssignment != null) {
        sidToHostname.putAll(oldAssignment.getNodeHost());
    }
    for (Entry<String, SupervisorInfo> entry : cluster.entrySet()) {
        String supervisorId = entry.getKey();
        SupervisorInfo supervisorInfo = entry.getValue();
        sidToHostname.put(supervisorId, supervisorInfo.getHostName());
    }
    return sidToHostname;
}

Example 8 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class DefaultTopologyScheduler method freeUsed.

/**
     * @@@ Here maybe exist one problem, some dead slots have been free
     * 
     * @param context
     */
protected void freeUsed(TopologyAssignContext context) {
    Set<Integer> canFree = new HashSet<Integer>();
    canFree.addAll(context.getAllTaskIds());
    canFree.removeAll(context.getUnstoppedTaskIds());
    Map<String, SupervisorInfo> cluster = context.getCluster();
    Assignment oldAssigns = context.getOldAssignment();
    for (Integer task : canFree) {
        ResourceWorkerSlot worker = oldAssigns.getWorkerByTaskId(task);
        if (worker == null) {
            LOG.warn("When free rebalance resource, no ResourceAssignment of task " + task);
            continue;
        }
        SupervisorInfo supervisorInfo = cluster.get(worker.getNodeId());
        if (supervisorInfo == null) {
            continue;
        }
        supervisorInfo.getAvailableWorkerPorts().add(worker.getPort());
    }
}

Example 9 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class WorkerScheduler method getAvailableWorkers.

public List<ResourceWorkerSlot> getAvailableWorkers(DefaultTopologyAssignContext context, Set<Integer> needAssign, int allocWorkerNum) {
    int workersNum = getAvailableWorkersNum(context);
    if (workersNum < allocWorkerNum) {
        throw new FailedAssignTopologyException("there's no enough worker. allocWorkerNum=" + allocWorkerNum + ", availableWorkerNum=" + workersNum);
    }
    workersNum = allocWorkerNum;
    List<ResourceWorkerSlot> assignedWorkers = new ArrayList<ResourceWorkerSlot>();
    // userdefine assignments, but dont't try to use custom scheduling for
    // TM bolts now.
    getRightWorkers(context, needAssign, assignedWorkers, workersNum, getUserDefineWorkers(context, ConfigExtension.getUserDefineAssignment(context.getStormConf())));
    // old assignments
    if (ConfigExtension.isUseOldAssignment(context.getStormConf())) {
        getRightWorkers(context, needAssign, assignedWorkers, workersNum, context.getOldWorkers());
    } else if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_REBALANCE && context.isReassign() == false) {
        int cnt = 0;
        for (ResourceWorkerSlot worker : context.getOldWorkers()) {
            if (cnt < workersNum) {
                ResourceWorkerSlot resFreeWorker = new ResourceWorkerSlot();
                resFreeWorker.setPort(worker.getPort());
                resFreeWorker.setHostname(worker.getHostname());
                resFreeWorker.setNodeId(worker.getNodeId());
                assignedWorkers.add(resFreeWorker);
                cnt++;
            } else {
                break;
            }
        }
    }
    // calculate rest TM bolts
    int workersForSingleTM = 0;
    if (context.getAssignSingleWorkerForTM()) {
        for (Integer taskId : needAssign) {
            String componentName = context.getTaskToComponent().get(taskId);
            if (componentName.equals(Common.TOPOLOGY_MASTER_COMPONENT_ID)) {
                workersForSingleTM++;
            }
        }
    }
    LOG.info("Get workers from user define and old assignments: " + assignedWorkers);
    int restWokerNum = workersNum - assignedWorkers.size();
    if (restWokerNum < 0)
        throw new FailedAssignTopologyException("Too much workers are needed for user define or old assignments. workersNum=" + workersNum + ", assignedWokersNum=" + assignedWorkers.size());
    for (int i = 0; i < restWokerNum; i++) {
        assignedWorkers.add(new ResourceWorkerSlot());
    }
    List<SupervisorInfo> isolationSupervisors = this.getIsolationSupervisors(context);
    if (isolationSupervisors.size() != 0) {
        putAllWorkerToSupervisor(assignedWorkers, getResAvailSupervisors(isolationSupervisors));
    } else {
        putAllWorkerToSupervisor(assignedWorkers, getResAvailSupervisors(context.getCluster()));
    }
    this.setAllWorkerMemAndCpu(context.getStormConf(), assignedWorkers);
    LOG.info("Assigned workers=" + assignedWorkers);
    return assignedWorkers;
}

Example 10 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class WorkerScheduler method putWorkerToSupervisor.

private void putWorkerToSupervisor(List<ResourceWorkerSlot> assignedWorkers, List<SupervisorInfo> supervisors) {
    int allUsedPorts = 0;
    for (SupervisorInfo supervisor : supervisors) {
        int supervisorUsedPorts = supervisor.getWorkerPorts().size() - supervisor.getAvailableWorkerPorts().size();
        allUsedPorts = allUsedPorts + supervisorUsedPorts;
    }
    // per supervisor should be allocated ports in theory
    int theoryAveragePorts = (allUsedPorts + assignedWorkers.size()) / supervisors.size() + 1;
    // supervisor which use more than theoryAveragePorts ports will be
    // pushed overLoadSupervisors
    List<SupervisorInfo> overLoadSupervisors = new ArrayList<SupervisorInfo>();
    int key = 0;
    Iterator<ResourceWorkerSlot> iterator = assignedWorkers.iterator();
    while (iterator.hasNext()) {
        if (supervisors.size() == 0)
            break;
        if (key >= supervisors.size())
            key = 0;
        SupervisorInfo supervisor = supervisors.get(key);
        int supervisorUsedPorts = supervisor.getWorkerPorts().size() - supervisor.getAvailableWorkerPorts().size();
        if (supervisorUsedPorts < theoryAveragePorts) {
            ResourceWorkerSlot worker = iterator.next();
            if (worker.getNodeId() != null)
                continue;
            worker.setHostname(supervisor.getHostName());
            worker.setNodeId(supervisor.getSupervisorId());
            worker.setPort(supervisor.getAvailableWorkerPorts().iterator().next());
            supervisor.getAvailableWorkerPorts().remove(worker.getPort());
            if (supervisor.getAvailableWorkerPorts().size() == 0)
                supervisors.remove(supervisor);
            key++;
        } else {
            overLoadSupervisors.add(supervisor);
            supervisors.remove(supervisor);
        }
    }
    // rest assignedWorkers will be allocate supervisor by deal
    Collections.sort(overLoadSupervisors, new Comparator<SupervisorInfo>() {

        @Override
        public int compare(SupervisorInfo o1, SupervisorInfo o2) {
            // TODO Auto-generated method stub
            return -NumberUtils.compare(o1.getAvailableWorkerPorts().size(), o2.getAvailableWorkerPorts().size());
        }
    });
    key = 0;
    while (iterator.hasNext()) {
        if (overLoadSupervisors.size() == 0)
            break;
        if (key >= overLoadSupervisors.size())
            key = 0;
        ResourceWorkerSlot worker = iterator.next();
        if (worker.getNodeId() != null)
            continue;
        SupervisorInfo supervisor = overLoadSupervisors.get(key);
        worker.setHostname(supervisor.getHostName());
        worker.setNodeId(supervisor.getSupervisorId());
        worker.setPort(supervisor.getAvailableWorkerPorts().iterator().next());
        supervisor.getAvailableWorkerPorts().remove(worker.getPort());
        if (supervisor.getAvailableWorkerPorts().size() == 0)
            overLoadSupervisors.remove(supervisor);
        key++;
    }
}