Examples with StormClusterState com.alibaba.jstorm.cluster.StormClusterState used on opensource projects

Example 1 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class DoRebalanceTransitionCallback method setTaskInfo.

private void setTaskInfo(StormTopology oldTopology, StormTopology newTopology) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    // Retrieve the max task ID
    TreeSet<Integer> taskIds = new TreeSet<Integer>(clusterState.task_ids(topologyid));
    int cnt = taskIds.descendingIterator().next();
    cnt = setBoltInfo(oldTopology, newTopology, cnt, clusterState);
    cnt = setSpoutInfo(oldTopology, newTopology, cnt, clusterState);
}

Example 2 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class JStormUtils method isKilledStatus.

public static boolean isKilledStatus(TopologyContext topologyContext) {
    boolean ret = false;
    StormClusterState zkCluster = topologyContext.getZkCluster();
    String topologyId = topologyContext.getTopologyId();
    try {
        StormBase stormBase = zkCluster.storm_base(topologyId, null);
        boolean isKilledStatus = stormBase != null && stormBase.getStatus().getStatusType().equals(StatusType.killed);
        ret = (stormBase == null || isKilledStatus);
    } catch (Exception e) {
        LOG.warn("Failed to get stormBase", e);
    }
    return ret;
}

Example 3 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class Supervisor method mkSupervisor.

//volatile HealthStatus healthStatus = new HealthStatus();
/**
     * create and start a supervisor
     *
     * @param conf          : configuration (default.yaml & storm.yaml)
     * @param sharedContext : null (right now)
     * @return SupervisorManger: which is used to shutdown all workers and supervisor
     */
@SuppressWarnings("rawtypes")
public SupervisorManger mkSupervisor(Map conf, IContext sharedContext) throws Exception {
    LOG.info("Starting Supervisor with conf " + conf);
    /**
         * Step 1: cleanup all files in /storm-local-dir/supervisor/tmp
         */
    String path = StormConfig.supervisorTmpDir(conf);
    FileUtils.cleanDirectory(new File(path));
    /**
         * Step 2: create ZK operation instance StormClusterState
         */
    StormClusterState stormClusterState = Cluster.mk_storm_cluster_state(conf);
    String hostName = JStormServerUtils.getHostName(conf);
    WorkerReportError workerReportError = new WorkerReportError(stormClusterState, hostName);
    /**
         * Step 3, create LocalStat (a simple KV store)
         * 3.1 create LocalState instance;
         * 3.2 get supervisorId, if there's no supervisorId, create one
         */
    LocalState localState = StormConfig.supervisorState(conf);
    String supervisorId = (String) localState.get(Common.LS_ID);
    if (supervisorId == null) {
        supervisorId = UUID.randomUUID().toString();
        localState.put(Common.LS_ID, supervisorId);
    }
    //clean LocalStat's zk-assignment & versions
    localState.remove(Common.LS_LOCAl_ZK_ASSIGNMENTS);
    localState.remove(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION);
    Vector<AsyncLoopThread> threads = new Vector<>();
    // Step 4 create HeartBeat
    // every supervisor.heartbeat.frequency.secs, write SupervisorInfo to ZK
    // sync heartbeat to nimbus
    Heartbeat hb = new Heartbeat(conf, stormClusterState, supervisorId, localState, checkStatus);
    hb.update();
    AsyncLoopThread heartbeat = new AsyncLoopThread(hb, false, null, Thread.MIN_PRIORITY, true);
    threads.add(heartbeat);
    // Sync heartbeat to Apsara Container
    AsyncLoopThread syncContainerHbThread = SyncContainerHb.mkSupervisorInstance(conf);
    if (syncContainerHbThread != null) {
        threads.add(syncContainerHbThread);
    }
    // Step 5 create and start sync Supervisor thread
    // every supervisor.monitor.frequency.secs second run SyncSupervisor
    ConcurrentHashMap<String, String> workerThreadPids = new ConcurrentHashMap<>();
    SyncProcessEvent syncProcessEvent = new SyncProcessEvent(supervisorId, conf, localState, workerThreadPids, sharedContext, workerReportError);
    EventManagerImp syncSupEventManager = new EventManagerImp();
    AsyncLoopThread syncSupEventThread = new AsyncLoopThread(syncSupEventManager);
    threads.add(syncSupEventThread);
    SyncSupervisorEvent syncSupervisorEvent = new SyncSupervisorEvent(supervisorId, conf, syncSupEventManager, stormClusterState, localState, syncProcessEvent, hb);
    int syncFrequency = JStormUtils.parseInt(conf.get(Config.SUPERVISOR_MONITOR_FREQUENCY_SECS));
    EventManagerPusher syncSupervisorPusher = new EventManagerPusher(syncSupEventManager, syncSupervisorEvent, syncFrequency);
    AsyncLoopThread syncSupervisorThread = new AsyncLoopThread(syncSupervisorPusher);
    threads.add(syncSupervisorThread);
    // Step 6 start httpserver
    Httpserver httpserver = null;
    if (!StormConfig.local_mode(conf)) {
        int port = ConfigExtension.getSupervisorDeamonHttpserverPort(conf);
        httpserver = new Httpserver(port, conf);
        httpserver.start();
    }
    //Step 7 check supervisor
    if (!StormConfig.local_mode(conf)) {
        if (ConfigExtension.isEnableCheckSupervisor(conf)) {
            SupervisorHealth supervisorHealth = new SupervisorHealth(conf, checkStatus, supervisorId);
            AsyncLoopThread healthThread = new AsyncLoopThread(supervisorHealth, false, null, Thread.MIN_PRIORITY, true);
            threads.add(healthThread);
        }
        // init refresh config thread
        AsyncLoopThread refreshConfigThread = new AsyncLoopThread(new SupervisorRefreshConfig(conf));
        threads.add(refreshConfigThread);
    }
    // create SupervisorManger which can shutdown all supervisor and workers
    return new SupervisorManger(conf, supervisorId, threads, syncSupEventManager, httpserver, stormClusterState, workerThreadPids);
}

Example 4 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class ServiceHandler method setupStormCode.

/**
     * create local topology files in blobstore and sync metadata to zk
     */
private void setupStormCode(Map<Object, Object> conf, String topologyId, String tmpJarLocation, Map<Object, Object> stormConf, StormTopology topology) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    String codeKey = StormConfig.master_stormcode_key(topologyId);
    String confKey = StormConfig.master_stormconf_key(topologyId);
    // in local mode there is no jar
    if (tmpJarLocation != null) {
        setupJar(tmpJarLocation, topologyId, blobStore, clusterState, nimbusInfo, false);
    }
    blobStore.createBlob(confKey, Utils.serialize(stormConf), new SettableBlobMeta());
    blobStore.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta());
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        clusterState.setup_blobstore(codeKey, nimbusInfo, BlobStoreUtils.getVersionForKey(codeKey, nimbusInfo, conf));
    }
    LOG.info("Successfully create blobstore for topology {}", topologyId);
}

Example 5 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class ServiceHandler method submitTopologyWithOpts.

/**
     * Submit one Topology
     *
     * @param topologyName        String: topology name
     * @param uploadedJarLocation String: already uploaded jar path
     * @param jsonConf            String: jsonConf serialize all toplogy configuration to
     *                            Json
     * @param topology            StormTopology: topology Object
     */
@SuppressWarnings("unchecked")
@Override
public String submitTopologyWithOpts(String topologyName, String uploadedJarLocation, String jsonConf, StormTopology topology, SubmitOptions options) throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException, TException {
    LOG.info("Receive " + topologyName + ", uploadedJarLocation:" + uploadedJarLocation);
    long start = System.nanoTime();
    //check topologyname is valid
    if (!Common.charValidate(topologyName)) {
        throw new InvalidTopologyException(topologyName + " is not a valid topology name");
    }
    Map<Object, Object> serializedConf = (Map<Object, Object>) JStormUtils.from_json(jsonConf);
    if (serializedConf == null) {
        LOG.warn("Failed to serialized Configuration");
        throw new InvalidTopologyException("Failed to serialize topology configuration");
    }
    Common.confValidate(serializedConf, data.getConf());
    boolean enableDeploy = ConfigExtension.getTopologyHotDeplogyEnable(serializedConf);
    try {
        checkTopologyActive(data, topologyName, enableDeploy);
    } catch (AlreadyAliveException e) {
        LOG.info(topologyName + " already exists ");
        throw e;
    } catch (NotAliveException e) {
        LOG.info(topologyName + " is not alive ");
        throw e;
    } catch (Throwable e) {
        LOG.info("Failed to check whether topology is alive or not", e);
        throw new TException(e);
    }
    if (enableDeploy) {
        LOG.info("deploy the topology");
        try {
            StormClusterState stormClusterState = data.getStormClusterState();
            String topologyId = Cluster.get_topology_id(stormClusterState, topologyName);
            if (topologyId == null) {
                throw new NotAliveException(topologyName);
            }
            LOG.info("start kill the old  topology {}", topologyId);
            Map oldConf = new HashMap();
            oldConf.putAll(conf);
            Map killedStormConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
            if (killedStormConf != null) {
                oldConf.putAll(killedStormConf);
            }
            NimbusUtils.transitionName(data, topologyName, true, StatusType.kill, 0);
            KillTopologyEvent.pushEvent(topologyId);
            notifyTopologyActionListener(topologyName, "killTopology");
            //wait all workers' are killed
            final long timeoutSeconds = ConfigExtension.getTaskCleanupTimeoutSec(oldConf);
            ConcurrentHashMap<String, Semaphore> topologyIdtoSem = data.getTopologyIdtoSem();
            if (!topologyIdtoSem.contains(topologyId)) {
                topologyIdtoSem.putIfAbsent(topologyId, new Semaphore(0));
            }
            Semaphore semaphore = topologyIdtoSem.get(topologyId);
            if (semaphore != null) {
                semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS);
                topologyIdtoSem.remove(semaphore);
            }
            LOG.info("success kill the old topology {}", topologyId);
        } catch (Exception e) {
            String errMsg = "Failed to kill topology " + topologyName;
            LOG.error(errMsg, e);
            throw new TException(errMsg);
        }
    }
    String topologyId = null;
    synchronized (data) {
        // avoid same topologies from being submitted at the same time
        Set<String> pendingTopologies = data.getPendingSubmitTopologies().buildMap().keySet();
        for (String cachTopologyId : pendingTopologies) {
            if (cachTopologyId.contains(topologyName + "-"))
                throw new AlreadyAliveException(topologyName + "  were submitted");
        }
        int counter = data.getSubmittedCount().incrementAndGet();
        topologyId = Common.topologyNameToId(topologyName, counter);
        data.getPendingSubmitTopologies().put(topologyId, null);
    }
    try {
        serializedConf.put(Config.TOPOLOGY_ID, topologyId);
        serializedConf.put(Config.TOPOLOGY_NAME, topologyName);
        Map<Object, Object> stormConf;
        stormConf = NimbusUtils.normalizeConf(conf, serializedConf, topology);
        LOG.info("Normalized configuration:" + stormConf);
        Map<Object, Object> totalStormConf = new HashMap<Object, Object>(conf);
        totalStormConf.putAll(stormConf);
        StormTopology normalizedTopology = NimbusUtils.normalizeTopology(stormConf, topology, true);
        // this validates the structure of the topology
        Common.validate_basic(normalizedTopology, totalStormConf, topologyId);
        // don't need generate real topology, so skip Common.system_topology
        // Common.system_topology(totalStormConf, topology);
        StormClusterState stormClusterState = data.getStormClusterState();
        // create /local-dir/nimbus/topologyId/xxxx files
        setupStormCode(conf, topologyId, uploadedJarLocation, stormConf, normalizedTopology);
        // wait for blob replication before activate topology
        waitForDesiredCodeReplication(conf, topologyId);
        // generate TaskInfo for every bolt or spout in ZK
        // /ZK/tasks/topoologyId/xxx
        setupZkTaskInfo(conf, topologyId, stormClusterState);
        //mkdir topology error directory
        String path = Cluster.taskerror_storm_root(topologyId);
        stormClusterState.mkdir(path);
        // make assignments for a topology
        LOG.info("Submit for " + topologyName + " with conf " + serializedConf);
        makeAssignment(topologyName, topologyId, options.get_initial_status());
        // push start event after startup
        double metricsSampleRate = ConfigExtension.getMetricSampleRate(stormConf);
        StartTopologyEvent.pushEvent(topologyId, metricsSampleRate);
        notifyTopologyActionListener(topologyName, "submitTopology");
    } catch (FailedAssignTopologyException e) {
        StringBuilder sb = new StringBuilder();
        sb.append("Fail to sumbit topology, Root cause:");
        if (e.getMessage() == null) {
            sb.append("submit timeout");
        } else {
            sb.append(e.getMessage());
        }
        sb.append("\n\n");
        sb.append("topologyId:" + topologyId);
        sb.append(", uploadedJarLocation:" + uploadedJarLocation + "\n");
        LOG.error(sb.toString(), e);
        throw new TopologyAssignException(sb.toString());
    } catch (InvalidParameterException e) {
        StringBuilder sb = new StringBuilder();
        sb.append("Fail to sumbit topology ");
        sb.append(e.getMessage());
        sb.append(", cause:" + e.getCause());
        sb.append("\n\n");
        sb.append("topologyId:" + topologyId);
        sb.append(", uploadedJarLocation:" + uploadedJarLocation + "\n");
        LOG.error(sb.toString(), e);
        throw new InvalidParameterException(sb.toString());
    } catch (InvalidTopologyException e) {
        LOG.error("Topology is invalid. " + e.get_msg());
        throw e;
    } catch (Throwable e) {
        StringBuilder sb = new StringBuilder();
        sb.append("Fail to sumbit topology ");
        sb.append(e.getMessage());
        sb.append(", cause:" + e.getCause());
        sb.append("\n\n");
        sb.append("topologyId:" + topologyId);
        sb.append(", uploadedJarLocation:" + uploadedJarLocation + "\n");
        LOG.error(sb.toString(), e);
        throw new TopologyAssignException(sb.toString());
    } finally {
        // when make assignment for a topology,so remove the topologyid form
        // pendingSubmitTopologys
        data.getPendingSubmitTopologies().remove(topologyId);
        double spend = (System.nanoTime() - start) / TimeUtils.NS_PER_US;
        SimpleJStormMetric.updateNimbusHistogram("submitTopologyWithOpts", spend);
        LOG.info("submitTopologyWithOpts {} costs {}ms", topologyName, spend);
    }
    return topologyId;
}