Examples with ResourceWorkerSlot com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot used on opensource projects

Example 21 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method getTaskStartTimes.

public static Map<Integer, Integer> getTaskStartTimes(TopologyAssignContext context, NimbusData nimbusData, String topologyId, Assignment existingAssignment, Set<ResourceWorkerSlot> workers) throws Exception {
    Map<Integer, Integer> startTimes = new TreeMap<>();
    if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_NEW) {
        int nowSecs = TimeUtils.current_time_secs();
        for (ResourceWorkerSlot worker : workers) {
            for (Integer changedTaskId : worker.getTasks()) {
                startTimes.put(changedTaskId, nowSecs);
            }
        }
        return startTimes;
    }
    Set<ResourceWorkerSlot> oldWorkers = new HashSet<>();
    if (existingAssignment != null) {
        Map<Integer, Integer> taskStartTimeSecs = existingAssignment.getTaskStartTimeSecs();
        if (taskStartTimeSecs != null) {
            startTimes.putAll(taskStartTimeSecs);
        }
        if (existingAssignment.getWorkers() != null) {
            oldWorkers = existingAssignment.getWorkers();
        }
    }
    Set<Integer> changedTaskIds = getNewOrChangedTaskIds(oldWorkers, workers);
    int nowSecs = TimeUtils.current_time_secs();
    for (Integer changedTaskId : changedTaskIds) {
        startTimes.put(changedTaskId, nowSecs);
        NimbusUtils.removeTopologyTaskHb(nimbusData, topologyId, changedTaskId);
    }
    Set<Integer> removedTaskIds = getRemovedTaskIds(oldWorkers, workers);
    for (Integer removedTaskId : removedTaskIds) {
        startTimes.remove(removedTaskId);
        NimbusUtils.removeTopologyTaskHb(nimbusData, topologyId, removedTaskId);
    }
    LOG.info("Task assignment has been changed: " + changedTaskIds + ", removed tasks " + removedTaskIds);
    return startTimes;
}

Example 22 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class RefreshEvent method doRefreshTopologies.

/**
 * refresh metric settings of topologies and sync metric meta from local cache
 */
@SuppressWarnings("unchecked")
private void doRefreshTopologies() {
    for (String topology : JStormMetrics.SYS_TOPOLOGIES) {
        if (!context.getTopologyMetricContexts().containsKey(topology)) {
            LOG.info("adding {} to metric context.", topology);
            Map conf = new HashMap();
            if (topology.equals(JStormMetrics.CLUSTER_METRIC_KEY)) {
                // there's no need to consider sample rate when cluster metrics merge
                conf.put(ConfigExtension.TOPOLOGY_METRIC_SAMPLE_RATE, 1.0);
            }
            Set<ResourceWorkerSlot> workerSlot = Sets.newHashSet(new ResourceWorkerSlot());
            TopologyMetricContext metricContext = new TopologyMetricContext(topology, workerSlot, conf);
            context.getTopologyMetricContexts().putIfAbsent(topology, metricContext);
            syncMetaFromCache(topology, context.getTopologyMetricContexts().get(topology));
        }
        // in the first 10 min, sync every 1 min
        if (context.getNimbusData().uptime() < INIT_SYNC_REMOTE_META_TIME_SEC) {
            syncMetaFromRemote(topology, context.getTopologyMetricContexts().get(topology), Lists.newArrayList(MetaType.TOPOLOGY, MetaType.WORKER, MetaType.NIMBUS));
        } else {
            // after that, sync every 5 min
            checkTimeAndSyncMetaFromRemote(topology, context.getTopologyMetricContexts().get(topology), Lists.newArrayList(MetaType.values()));
        }
    }
    Map<String, Assignment> assignMap;
    try {
        assignMap = Cluster.get_all_assignment(context.getStormClusterState(), null);
        for (Entry<String, Assignment> entry : assignMap.entrySet()) {
            String topology = entry.getKey();
            Assignment assignment = entry.getValue();
            TopologyMetricContext metricContext = context.getTopologyMetricContexts().get(topology);
            if (metricContext == null) {
                metricContext = new TopologyMetricContext(assignment.getWorkers());
                metricContext.setTaskNum(NimbusUtils.getTopologyTaskNum(assignment));
                syncMetaFromCache(topology, metricContext);
                LOG.info("adding {} to metric context.", topology);
                context.getTopologyMetricContexts().put(topology, metricContext);
            } else {
                boolean modify = false;
                if (metricContext.getTaskNum() != NimbusUtils.getTopologyTaskNum(assignment)) {
                    modify = true;
                    metricContext.setTaskNum(NimbusUtils.getTopologyTaskNum(assignment));
                }
                if (!assignment.getWorkers().equals(metricContext.getWorkerSet())) {
                    modify = true;
                    metricContext.setWorkerSet(assignment.getWorkers());
                }
                // for normal topologies, only sync topology/component level metrics
                checkTimeAndSyncMetaFromRemote(topology, metricContext, Lists.newArrayList(MetaType.TOPOLOGY, MetaType.COMPONENT));
                // we may need to sync meta when task num/workers change
                metricContext.setSyncMeta(!modify);
            }
        }
    } catch (Exception e1) {
        LOG.warn("Failed to get assignments");
        return;
    }
    List<String> removing = new ArrayList<>();
    for (String topology : context.getTopologyMetricContexts().keySet()) {
        if (!JStormMetrics.SYS_TOPOLOGY_SET.contains(topology) && !assignMap.containsKey(topology)) {
            removing.add(topology);
        }
    }
    for (String topology : removing) {
        LOG.info("removing topology:{}", topology);
        RemoveTopologyEvent.pushEvent(topology);
        syncRemoteTimes.remove(topology);
    }
}

Example 23 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class ServiceHandler method getTopologyWorkersToSupervisorIds.

@Override
public Map<String, Map<String, String>> getTopologyWorkersToSupervisorIds(String topologyName) throws NotAliveException, TException {
    StormClusterState stormClusterState = data.getStormClusterState();
    String topologyId = getTopologyId(topologyName);
    Map<String, Map<String, String>> ret = new HashMap<>();
    try {
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        Set<ResourceWorkerSlot> workers = assignment.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String supervisorId = worker.getNodeId();
            if (!ret.containsKey(worker.getHostname()))
                ret.put(worker.getHostname(), new HashMap<String, String>());
            ret.get(worker.getHostname()).put(String.valueOf(worker.getPort()), supervisorId);
        }
    } catch (Exception ex) {
        LOG.error("Error:", ex);
    }
    return ret;
}

Example 24 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class ServiceHandler method getTopologyTasksToSupervisorIds.

@Override
public Map<Integer, String> getTopologyTasksToSupervisorIds(String topologyName) throws TException {
    StormClusterState stormClusterState = data.getStormClusterState();
    String topologyId = getTopologyId(topologyName);
    Map<Integer, String> ret = new HashMap<>();
    try {
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        Set<ResourceWorkerSlot> workers = assignment.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String supervisorId = worker.getNodeId();
            for (Integer task : worker.getTasks()) {
                ret.put(task, supervisorId);
            }
        }
    } catch (Exception ex) {
        LOG.error("Error:", ex);
    }
    return ret;
}

Example 25 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class SyncSupervisorEvent method readMyTasks.

/**
 * get local node's tasks
 *
 * @return Map: [port, LocalAssignment]
 */
@SuppressWarnings("unused")
private Map<Integer, LocalAssignment> readMyTasks(StormClusterState stormClusterState, String topologyId, String supervisorId, Assignment assignmentInfo) throws Exception {
    Map<Integer, LocalAssignment> portTasks = new HashMap<>();
    Set<ResourceWorkerSlot> workers = assignmentInfo.getWorkers();
    if (workers == null) {
        LOG.error("No worker found for assignment {}!", assignmentInfo);
        return portTasks;
    }
    for (ResourceWorkerSlot worker : workers) {
        if (!supervisorId.equals(worker.getNodeId())) {
            continue;
        }
        portTasks.put(worker.getPort(), new LocalAssignment(topologyId, worker.getTasks(), Common.topologyIdToName(topologyId), worker.getMemSize(), worker.getCpu(), worker.getJvm(), assignmentInfo.getTimeStamp()));
    }
    return portTasks;
}