Examples with StormClusterState com.alibaba.jstorm.cluster.StormClusterState used on opensource projects

Example 26 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class TopologyAssign method setTopologyStatus.

/**
     * start a topology: set active status of the topology
     */
public void setTopologyStatus(TopologyAssignEvent event) throws Exception {
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    String topologyId = event.getTopologyId();
    String topologyName = event.getTopologyName();
    String group = event.getGroup();
    StormStatus status = new StormStatus(StatusType.active);
    if (event.getOldStatus() != null) {
        status = event.getOldStatus();
    }
    boolean isEnable = ConfigExtension.isEnablePerformanceMetrics(nimbusData.getConf());
    StormBase stormBase = stormClusterState.storm_base(topologyId, null);
    if (stormBase == null) {
        stormBase = new StormBase(topologyName, TimeUtils.current_time_secs(), status, group);
        stormBase.setEnableMonitor(isEnable);
        stormClusterState.activate_storm(topologyId, stormBase);
    } else {
        stormClusterState.update_storm(topologyId, status);
        stormClusterState.set_storm_monitor(topologyId, isEnable);
        // when monitor/rebalance/startup topologyName is null
        if (topologyName == null) {
            event.setTopologyName(stormBase.getStormName());
        }
    }
    LOG.info("Update " + topologyId + " " + status);
}

Example 27 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class NimbusUtils method updateTopologyTaskHb.

public static void updateTopologyTaskHb(NimbusData data, String topologyId) {
    StormClusterState clusterState = data.getStormClusterState();
    TopologyTaskHbInfo topologyTaskHb = null;
    try {
        topologyTaskHb = clusterState.topology_heartbeat(topologyId);
    } catch (Exception e) {
        LOG.error("updateTopologyTaskHb: Failed to get topology task heartbeat info", e);
    }
    if (topologyTaskHb != null) {
        data.getTasksHeartbeat().put(topologyId, topologyTaskHb);
    }
}

Example 28 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class NimbusUtils method cleanupCorruptTopologies.

/**
     * clean the topology which is in ZK but not in local dir
     *
     * @throws Exception
     */
public static void cleanupCorruptTopologies(NimbusData data) throws Exception {
    StormClusterState stormClusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    // we have only topology relative files , so we don't need filter
    Set<String> code_ids = Sets.newHashSet(BlobStoreUtils.code_ids(blobStore.listKeys()));
    // get topology in ZK /storms
    Set<String> active_ids = Sets.newHashSet(data.getStormClusterState().active_storms());
    //get topology in zk by blobs
    Set<String> blobsIdsOnZk = Sets.newHashSet(data.getStormClusterState().blobstore(null));
    Set<String> topologyIdsOnZkbyBlobs = BlobStoreUtils.code_ids(blobsIdsOnZk.iterator());
    Set<String> corrupt_ids = Sets.difference(active_ids, code_ids);
    Set<String> redundantIds = Sets.difference(topologyIdsOnZkbyBlobs, code_ids);
    Set<String> unionIds = Sets.union(corrupt_ids, redundantIds);
    // clean the topology which is in ZK but not in local dir
    for (String corrupt : unionIds) {
        LOG.info("Corrupt topology {} has state on zookeeper but doesn't have a local dir on Nimbus. Cleaning up...", corrupt);
        stormClusterState.remove_storm(corrupt);
        if (blobStore instanceof LocalFsBlobStore) {
            List<String> blobKeys = BlobStoreUtils.getKeyListFromId(data, corrupt);
            for (String key : blobKeys) {
                stormClusterState.remove_blobstore_key(key);
                stormClusterState.remove_key_version(key);
            }
        }
    }
    LOG.info("Successfully cleanup all old toplogies");
}

Example 29 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class TopologyAssign method mkAssignment.

/**
     * make assignments for a topology The nimbus core function, this function has been totally rewrite
     * 
     * @throws Exception
     */
public Assignment mkAssignment(TopologyAssignEvent event) throws Exception {
    String topologyId = event.getTopologyId();
    LOG.info("Determining assignment for " + topologyId);
    TopologyAssignContext context = prepareTopologyAssign(event);
    Set<ResourceWorkerSlot> assignments = null;
    if (!StormConfig.local_mode(nimbusData.getConf())) {
        IToplogyScheduler scheduler = schedulers.get(DEFAULT_SCHEDULER_NAME);
        assignments = scheduler.assignTasks(context);
    } else {
        assignments = mkLocalAssignment(context);
    }
    Assignment assignment = null;
    if (assignments != null && assignments.size() > 0) {
        Map<String, String> nodeHost = getTopologyNodeHost(context.getCluster(), context.getOldAssignment(), assignments);
        Map<Integer, Integer> startTimes = getTaskStartTimes(context, nimbusData, topologyId, context.getOldAssignment(), assignments);
        String codeDir = (String) nimbusData.getConf().get(Config.STORM_LOCAL_DIR);
        assignment = new Assignment(codeDir, assignments, nodeHost, startTimes);
        //  the topology binary changed.
        if (event.isScaleTopology()) {
            assignment.setAssignmentType(Assignment.AssignmentType.ScaleTopology);
        }
        StormClusterState stormClusterState = nimbusData.getStormClusterState();
        stormClusterState.set_assignment(topologyId, assignment);
        // update task heartbeat's start time
        NimbusUtils.updateTaskHbStartTime(nimbusData, assignment, topologyId);
        // @@@ TODO
        // Update metrics information in ZK when rebalance or reassignment
        // Only update metrics monitor status when creating topology
        // if (context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_REBALANCE
        // || context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_MONITOR)
        // NimbusUtils.updateMetricsInfo(nimbusData, topologyId, assignment);
        NimbusUtils.updateTopologyTaskTimeout(nimbusData, topologyId);
        LOG.info("Successfully make assignment for topology id " + topologyId + ": " + assignment);
    }
    return assignment;
}

Example 30 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class JStormHealthReporter method run.

@Override
public void run() {
    StormClusterState clusterState = workerData.getZkCluster();
    String topologyId = workerData.getTopologyId();
    Map<Integer, HealthCheckRegistry> taskHealthCheckMap = JStormHealthCheck.getTaskhealthcheckmap();
    int cnt = 0;
    for (Map.Entry<Integer, HealthCheckRegistry> entry : taskHealthCheckMap.entrySet()) {
        Integer taskId = entry.getKey();
        Map<String, Result> results = entry.getValue().runHealthChecks();
        for (Map.Entry<String, Result> result : results.entrySet()) {
            if (!result.getValue().isHealthy()) {
                try {
                    clusterState.report_task_error(topologyId, taskId, result.getValue().getMessage(), ErrorConstants.WARN, ErrorConstants.CODE_QUEUE_FULL, ErrorConstants.DURATION_SECS_QUEUE_FULL);
                    cnt++;
                } catch (Exception e) {
                    LOG.error("Failed to update health data in ZK for topo-{} task-{}.", topologyId, taskId, e);
                }
            }
        }
    }
    if (cnt > 0) {
        LOG.info("Successfully updated {} health data to ZK for topology:{}", cnt, topologyId);
    }
}