Examples with StormClusterState com.alibaba.jstorm.cluster.StormClusterState used on opensource projects

Example 11 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class FollowerRunnable method checkOwnMaster.

/**
     * Check whether current node is master or not
     * 
     * @throws Exception
     */
private void checkOwnMaster() throws Exception {
    int retry_times = 10;
    StormClusterState zkClient = data.getStormClusterState();
    for (int i = 0; i < retry_times; i++, JStormUtils.sleepMs(sleepTime)) {
        if (zkClient.leader_existed() == false) {
            continue;
        }
        String zkHost = zkClient.get_leader_host();
        if (hostPort.equals(zkHost) == true) {
            // current process own master
            return;
        }
        LOG.warn("Current Nimbus has start thrift, but fail to own zk master :" + zkHost);
    }
    // current process doesn't own master
    String err = "Current Nimubs fail to own nimbus_master, should halt process";
    LOG.error(err);
    JStormUtils.halt_process(0, err);
}

Example 12 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class FollowerRunnable method blobSync.

private synchronized void blobSync() {
    if (!data.isLeader()) {
        try {
            BlobStore blobStore = data.getBlobStore();
            StormClusterState clusterState = data.getStormClusterState();
            Set<String> localKeys = Sets.newHashSet(blobStore.listKeys());
            Set<String> zkKeys = Sets.newHashSet(clusterState.blobstore(blobSyncCallback));
            BlobSynchronizer blobSynchronizer = new BlobSynchronizer(blobStore, data.getConf());
            blobSynchronizer.setNimbusInfo(data.getNimbusHostPortInfo());
            blobSynchronizer.setBlobStoreKeySet(localKeys);
            blobSynchronizer.setZookeeperKeySet(zkKeys);
            blobSynchronizer.syncBlobs();
        } catch (Exception e) {
            LOG.error("blob sync error", e);
        }
    }
}

Example 13 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class FollowerRunnable method setupBlobstore.

// sets up blobstore state for all current keys
private void setupBlobstore() throws Exception {
    BlobStore blobStore = data.getBlobStore();
    StormClusterState clusterState = data.getStormClusterState();
    Set<String> localSetOfKeys = Sets.newHashSet(blobStore.listKeys());
    Set<String> allKeys = Sets.newHashSet(clusterState.active_keys());
    Set<String> localAvailableActiveKeys = Sets.intersection(localSetOfKeys, allKeys);
    // keys on local but not on zk, we will delete it
    Set<String> keysToDelete = Sets.difference(localSetOfKeys, allKeys);
    LOG.debug("deleting keys not on the zookeeper {}", keysToDelete);
    for (String key : keysToDelete) {
        blobStore.deleteBlob(key);
    }
    //    (log-debug "Creating list of key entries for blobstore inside zookeeper" all-keys "local" locally-available-active-keys)
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", allKeys, localAvailableActiveKeys);
    for (String key : localAvailableActiveKeys) {
        int versionForKey = BlobStoreUtils.getVersionForKey(key, data.getNimbusHostPortInfo(), data.getConf());
        clusterState.setup_blobstore(key, data.getNimbusHostPortInfo(), versionForKey);
    }
}

Example 14 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class FollowerRunnable method update_nimbus_detail.

private int update_nimbus_detail() throws Exception {
    //update count = count of zk's binary files - count of nimbus's binary files
    StormClusterState zkClusterState = data.getStormClusterState();
    // if we use other blobstore, such as HDFS, all nimbus slave can be leader
    // but if we use local blobstore, we should count topologies files
    int diffCount = 0;
    if (data.getBlobStore() instanceof LocalFsBlobStore) {
        Set<String> keysOnZk = Sets.newHashSet(zkClusterState.active_keys());
        Set<String> keysOnLocal = Sets.newHashSet(data.getBlobStore().listKeys());
        // we count number of keys which is on zk but not on local
        diffCount = Sets.difference(keysOnZk, keysOnLocal).size();
    }
    Map mtmp = zkClusterState.get_nimbus_detail(hostPort, false);
    if (mtmp == null) {
        mtmp = new HashMap();
    }
    mtmp.put(NIMBUS_DIFFER_COUNT_ZK, diffCount);
    zkClusterState.update_nimbus_detail(hostPort, mtmp);
    LOG.debug("update nimbus's detail " + mtmp);
    return diffCount;
}

Example 15 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class FollowerRunnable method check_nimbus_priority.

/**
     * Compared with other nimbus ,get priority of this nimbus
     *
     * @throws Exception
     */
private boolean check_nimbus_priority() throws Exception {
    int gap = update_nimbus_detail();
    if (gap == 0) {
        return true;
    }
    int left = SLAVE_NIMBUS_WAIT_TIME;
    while (left > 0) {
        LOG.info("nimbus.differ.count.zk is {}, so after {} seconds, nimbus will try to be Leader!", gap, left);
        Thread.sleep(10 * 1000);
        left -= 10;
    }
    StormClusterState zkClusterState = data.getStormClusterState();
    List<String> followers = zkClusterState.list_dirs(Cluster.NIMBUS_SLAVE_DETAIL_SUBTREE, false);
    if (followers == null || followers.size() == 0) {
        return false;
    }
    for (String follower : followers) {
        if (follower != null && !follower.equals(hostPort)) {
            Map bMap = zkClusterState.get_nimbus_detail(follower, false);
            if (bMap != null) {
                Object object = bMap.get(NIMBUS_DIFFER_COUNT_ZK);
                if (object != null && (JStormUtils.parseInt(object)) < gap) {
                    LOG.info("Current node can't be leader, due to {} has higher priority", follower);
                    return false;
                }
            }
        }
    }
    return true;
}