Examples with StormBase com.alibaba.jstorm.cluster.StormBase used on opensource projects

Example 1 with StormBase

use of com.alibaba.jstorm.cluster.StormBase in project jstorm by alibaba.

the class JStormUtils method isKilledStatus.

public static boolean isKilledStatus(TopologyContext topologyContext) {
    boolean ret = false;
    StormClusterState zkCluster = topologyContext.getZkCluster();
    String topologyId = topologyContext.getTopologyId();
    try {
        StormBase stormBase = zkCluster.storm_base(topologyId, null);
        boolean isKilledStatus = stormBase != null && stormBase.getStatus().getStatusType().equals(StatusType.killed);
        ret = (stormBase == null || isKilledStatus);
    } catch (Exception e) {
        LOG.warn("Failed to get stormBase", e);
    }
    return ret;
}

Example 2 with StormBase

use of com.alibaba.jstorm.cluster.StormBase in project jstorm by alibaba.

the class RefreshActive method run.

@Override
public void run() {
    try {
        StatusType newTopologyStatus;
        // /ZK-DIR/topology
        StormBase base = zkCluster.storm_base(topologyId, this);
        if (base == null) {
            // normally the topology has been removed
            LOG.warn("Failed to get StormBase from ZK of " + topologyId);
            newTopologyStatus = StatusType.killed;
        } else {
            newTopologyStatus = base.getStatus().getStatusType();
        }
        // Process the topology status change
        StatusType oldTopologyStatus = workerData.getTopologyStatus();
        List<TaskShutdownDameon> tasks = workerData.getShutdownTasks();
        if (tasks == null) {
            LOG.info("Tasks aren't ready or are beginning to shutdown");
            return;
        }
        // If all connections were done, start to update topology status. Otherwise, just return.
        if (oldTopologyStatus == null) {
            if (!workerData.getWorkeInitConnectionStatus().get()) {
                return;
            }
        }
        if (oldTopologyStatus == null || !newTopologyStatus.equals(oldTopologyStatus)) {
            LOG.info("Old TopologyStatus:" + oldTopologyStatus + ", new TopologyStatus:" + newTopologyStatus);
            if (newTopologyStatus.equals(StatusType.active)) {
                for (TaskShutdownDameon task : tasks) {
                    if (task.getTask().getTaskStatus().isInit()) {
                        task.getTask().getTaskStatus().setStatus(TaskStatus.RUN);
                    } else {
                        task.active();
                    }
                }
            } else if (oldTopologyStatus == null || !oldTopologyStatus.equals(StatusType.inactive)) {
                for (TaskShutdownDameon task : tasks) {
                    if (task.getTask().getTaskStatus().isInit()) {
                        task.getTask().getTaskStatus().setStatus(TaskStatus.PAUSE);
                    } else {
                        task.deactive();
                    }
                }
            }
            workerData.setTopologyStatus(newTopologyStatus);
            if (base != null) {
                boolean newMonitorEnable = base.isEnableMonitor();
                boolean oldMonitorEnable = monitorEnable.get();
                if (newMonitorEnable != oldMonitorEnable) {
                    LOG.info("Change MonitorEnable from " + oldMonitorEnable + " to " + newMonitorEnable);
                    monitorEnable.set(newMonitorEnable);
                }
            }
        }
    } catch (Exception e) {
        LOG.error("Failed to get topology from ZK ", e);
    }
}

Example 3 with StormBase

use of com.alibaba.jstorm.cluster.StormBase in project jstorm by alibaba.

the class ServiceHandler method getTopologyInfo.

/**
     * Get TopologyInfo, it contain all data of the topology running status
     *
     * @return TopologyInfo
     */
@Override
public TopologyInfo getTopologyInfo(String topologyId) throws NotAliveException, TException {
    long start = System.nanoTime();
    StormClusterState stormClusterState = data.getStormClusterState();
    try {
        // get topology's StormBase
        StormBase base = stormClusterState.storm_base(topologyId, null);
        if (base == null) {
            throw new NotAliveException("No topology of " + topologyId);
        }
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        if (assignment == null) {
            throw new NotAliveException("No topology of " + topologyId);
        }
        TopologyTaskHbInfo topologyTaskHbInfo = data.getTasksHeartbeat().get(topologyId);
        Map<Integer, TaskHeartbeat> taskHbMap = null;
        if (topologyTaskHbInfo != null)
            taskHbMap = topologyTaskHbInfo.get_taskHbs();
        Map<Integer, TaskInfo> taskInfoMap = Cluster.get_all_taskInfo(stormClusterState, topologyId);
        Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, taskInfoMap);
        Map<Integer, String> taskToType = Cluster.get_all_task_type(stormClusterState, topologyId, taskInfoMap);
        String errorString;
        if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
            errorString = "Y";
        } else {
            errorString = "";
        }
        TopologySummary topologySummary = new TopologySummary();
        topologySummary.set_id(topologyId);
        topologySummary.set_name(base.getStormName());
        topologySummary.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
        topologySummary.set_status(base.getStatusString());
        topologySummary.set_numTasks(NimbusUtils.getTopologyTaskNum(assignment));
        topologySummary.set_numWorkers(assignment.getWorkers().size());
        topologySummary.set_errorInfo(errorString);
        Map<String, ComponentSummary> componentSummaryMap = new HashMap<String, ComponentSummary>();
        HashMap<String, List<Integer>> componentToTasks = JStormUtils.reverse_map(taskToComponent);
        for (Entry<String, List<Integer>> entry : componentToTasks.entrySet()) {
            String name = entry.getKey();
            List<Integer> taskIds = entry.getValue();
            if (taskIds == null || taskIds.size() == 0) {
                LOG.warn("No task of component " + name);
                continue;
            }
            ComponentSummary componentSummary = new ComponentSummary();
            componentSummaryMap.put(name, componentSummary);
            componentSummary.set_name(name);
            componentSummary.set_type(taskToType.get(taskIds.get(0)));
            componentSummary.set_parallel(taskIds.size());
            componentSummary.set_taskIds(taskIds);
        }
        Map<Integer, TaskSummary> taskSummaryMap = new TreeMap<Integer, TaskSummary>();
        Map<Integer, List<TaskError>> taskErrors = Cluster.get_all_task_errors(stormClusterState, topologyId);
        for (Integer taskId : taskInfoMap.keySet()) {
            TaskSummary taskSummary = new TaskSummary();
            taskSummaryMap.put(taskId, taskSummary);
            taskSummary.set_taskId(taskId);
            if (taskHbMap == null) {
                taskSummary.set_status("Starting");
                taskSummary.set_uptime(0);
            } else {
                TaskHeartbeat hb = taskHbMap.get(taskId);
                if (hb == null) {
                    taskSummary.set_status("Starting");
                    taskSummary.set_uptime(0);
                } else {
                    boolean isInactive = NimbusUtils.isTaskDead(data, topologyId, taskId);
                    if (isInactive)
                        taskSummary.set_status("INACTIVE");
                    else
                        taskSummary.set_status("ACTIVE");
                    taskSummary.set_uptime(hb.get_uptime());
                }
            }
            if (StringUtils.isBlank(errorString)) {
                continue;
            }
            List<TaskError> taskErrorList = taskErrors.get(taskId);
            if (taskErrorList != null && taskErrorList.size() != 0) {
                for (TaskError taskError : taskErrorList) {
                    ErrorInfo errorInfo = new ErrorInfo(taskError.getError(), taskError.getTimSecs(), taskError.getLevel(), taskError.getCode());
                    taskSummary.add_to_errors(errorInfo);
                    String component = taskToComponent.get(taskId);
                    componentSummaryMap.get(component).add_to_errors(errorInfo);
                }
            }
        }
        for (ResourceWorkerSlot workerSlot : assignment.getWorkers()) {
            String hostname = workerSlot.getHostname();
            int port = workerSlot.getPort();
            for (Integer taskId : workerSlot.getTasks()) {
                TaskSummary taskSummary = taskSummaryMap.get(taskId);
                taskSummary.set_host(hostname);
                taskSummary.set_port(port);
            }
        }
        TopologyInfo topologyInfo = new TopologyInfo();
        topologyInfo.set_topology(topologySummary);
        topologyInfo.set_components(JStormUtils.mk_list(componentSummaryMap.values()));
        topologyInfo.set_tasks(JStormUtils.mk_list(taskSummaryMap.values()));
        // return topology metric & component metric only
        List<MetricInfo> tpMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.TOPOLOGY);
        List<MetricInfo> compMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.COMPONENT);
        List<MetricInfo> workerMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
        MetricInfo taskMetric = MetricUtils.mkMetricInfo();
        MetricInfo streamMetric = MetricUtils.mkMetricInfo();
        MetricInfo nettyMetric = MetricUtils.mkMetricInfo();
        MetricInfo tpMetric, compMetric, workerMetric;
        if (tpMetricList == null || tpMetricList.size() == 0) {
            tpMetric = MetricUtils.mkMetricInfo();
        } else {
            // get the last min topology metric
            tpMetric = tpMetricList.get(tpMetricList.size() - 1);
        }
        if (compMetricList == null || compMetricList.size() == 0) {
            compMetric = MetricUtils.mkMetricInfo();
        } else {
            compMetric = compMetricList.get(0);
        }
        if (workerMetricList == null || workerMetricList.size() == 0) {
            workerMetric = MetricUtils.mkMetricInfo();
        } else {
            workerMetric = workerMetricList.get(0);
        }
        TopologyMetric topologyMetrics = new TopologyMetric(tpMetric, compMetric, workerMetric, taskMetric, streamMetric, nettyMetric);
        topologyInfo.set_metrics(topologyMetrics);
        return topologyInfo;
    } catch (TException e) {
        LOG.info("Failed to get topologyInfo " + topologyId, e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get topologyInfo " + topologyId, e);
        throw new TException("Failed to get topologyInfo" + topologyId);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getTopologyInfo", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 4 with StormBase

use of com.alibaba.jstorm.cluster.StormBase in project jstorm by alibaba.

the class StatusTransition method transitionLock.

/**
     * Changing status
     *
     * @param args -- will be used in the status changing callback
     */
public <T> void transitionLock(String topologyId, boolean errorOnNoTransition, StatusType changeStatus, T... args) throws Exception {
    // get ZK's topology node's data, which is StormBase
    StormBase stormbase = data.getStormClusterState().storm_base(topologyId, null);
    if (stormbase == null) {
        LOG.error("Cannot apply event: changing status " + topologyId + " -> " + changeStatus.getStatus() + ", cause: failed to get StormBase from ZK");
        return;
    }
    StormStatus currentStatus = stormbase.getStatus();
    if (currentStatus == null) {
        LOG.error("Cannot apply event: changing status " + topologyId + " -> " + changeStatus.getStatus() + ", cause: topologyStatus is null in ZK");
        return;
    }
    // <currentStatus, Map<changingStatus, callback>>
    Map<StatusType, Map<StatusType, Callback>> callbackMap = stateTransitions(topologyId, currentStatus);
    // get current changingCallbacks
    Map<StatusType, Callback> changingCallbacks = callbackMap.get(currentStatus.getStatusType());
    if (changingCallbacks == null || !changingCallbacks.containsKey(changeStatus) || changingCallbacks.get(changeStatus) == null) {
        String msg = "No transition for event: changing status:" + changeStatus.getStatus() + ", current status: " + currentStatus.getStatusType() + ", topology-id: " + topologyId;
        LOG.info(msg);
        if (errorOnNoTransition) {
            throw new RuntimeException(msg);
        }
        return;
    }
    Callback callback = changingCallbacks.get(changeStatus);
    Object obj = callback.execute(args);
    if (obj != null && obj instanceof StormStatus) {
        StormStatus newStatus = (StormStatus) obj;
        // update status to ZK
        data.getStormClusterState().update_storm(topologyId, newStatus);
        LOG.info("Successfully updated " + topologyId + " to status " + newStatus);
    }
    LOG.info("Successfully apply event: changing status " + topologyId + " -> " + changeStatus.getStatus());
}

Example 5 with StormBase

use of com.alibaba.jstorm.cluster.StormBase in project jstorm by alibaba.

the class NimbusUtils method getTopologySummary.

public static List<TopologySummary> getTopologySummary(StormClusterState stormClusterState, Map<String, Assignment> assignments) throws Exception {
    List<TopologySummary> topologySummaries = new ArrayList<TopologySummary>();
    // get all active topology's StormBase
    Map<String, StormBase> bases = Cluster.get_all_StormBase(stormClusterState);
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        String topologyId = entry.getKey();
        StormBase base = entry.getValue();
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        if (assignment == null) {
            LOG.error("Failed to get assignment of " + topologyId);
            continue;
        }
        assignments.put(topologyId, assignment);
        int num_workers = assignment.getWorkers().size();
        int num_tasks = getTopologyTaskNum(assignment);
        String errorString = null;
        if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
            errorString = "Y";
        } else {
            errorString = "";
        }
        TopologySummary topology = new TopologySummary();
        topology.set_id(topologyId);
        topology.set_name(base.getStormName());
        topology.set_status(base.getStatusString());
        topology.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
        topology.set_numWorkers(num_workers);
        topology.set_numTasks(num_tasks);
        topology.set_errorInfo(errorString);
        topologySummaries.add(topology);
    }
    return topologySummaries;
}