Examples with LocalFsBlobStore com.alibaba.jstorm.blobstore.LocalFsBlobStore used on opensource projects

Example 6 with LocalFsBlobStore

use of com.alibaba.jstorm.blobstore.LocalFsBlobStore in project jstorm by alibaba.

the class FollowerRunnable method run.

@Override
public void run() {
    // TODO Auto-generated method stub
    LOG.info("Follower Thread starts!");
    while (state) {
        StormClusterState zkClusterState = data.getStormClusterState();
        try {
            Thread.sleep(sleepTime);
            if (!zkClusterState.leader_existed()) {
                this.tryToBeLeader(data.getConf());
                continue;
            }
            String master = zkClusterState.get_leader_host();
            boolean isZkLeader = isLeader(master);
            if (isZkLeader) {
                if (!data.isLeader()) {
                    zkClusterState.unregister_nimbus_host(hostPort);
                    zkClusterState.unregister_nimbus_detail(hostPort);
                    data.setLeader(true);
                    leaderCallback.execute();
                }
                continue;
            } else {
                if (data.isLeader()) {
                    LOG.info("New ZK master is " + master);
                    JStormUtils.halt_process(1, "Lose ZK master node, halt process");
                    return;
                }
            }
            // here the nimbus is not leader
            if (data.getBlobStore() instanceof LocalFsBlobStore) {
                blobSync();
            }
            zkClusterState.update_nimbus_slave(hostPort, data.uptime());
            update_nimbus_detail();
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            continue;
        } catch (Exception e) {
            if (state) {
                LOG.error("Unknow exception ", e);
            }
        }
    }
    LOG.info("Follower Thread has closed!");
}

Example 7 with LocalFsBlobStore

use of com.alibaba.jstorm.blobstore.LocalFsBlobStore in project jstorm by alibaba.

the class FollowerRunnable method update_nimbus_detail.

private int update_nimbus_detail() throws Exception {
    //update count = count of zk's binary files - count of nimbus's binary files
    StormClusterState zkClusterState = data.getStormClusterState();
    // if we use other blobstore, such as HDFS, all nimbus slave can be leader
    // but if we use local blobstore, we should count topologies files
    int diffCount = 0;
    if (data.getBlobStore() instanceof LocalFsBlobStore) {
        Set<String> keysOnZk = Sets.newHashSet(zkClusterState.active_keys());
        Set<String> keysOnLocal = Sets.newHashSet(data.getBlobStore().listKeys());
        // we count number of keys which is on zk but not on local
        diffCount = Sets.difference(keysOnZk, keysOnLocal).size();
    }
    Map mtmp = zkClusterState.get_nimbus_detail(hostPort, false);
    if (mtmp == null) {
        mtmp = new HashMap();
    }
    mtmp.put(NIMBUS_DIFFER_COUNT_ZK, diffCount);
    zkClusterState.update_nimbus_detail(hostPort, mtmp);
    LOG.debug("update nimbus's detail " + mtmp);
    return diffCount;
}

Example 8 with LocalFsBlobStore

use of com.alibaba.jstorm.blobstore.LocalFsBlobStore in project jstorm by alibaba.

the class StormConfig method write_nimbus_topology_conf.

public static void write_nimbus_topology_conf(String topologyId, Map topoConf, NimbusData data) throws Exception {
    String confKey = master_stormconf_key(topologyId);
    AtomicOutputStream out = data.getBlobStore().updateBlob(confKey);
    out.write(Utils.serialize(topoConf));
    out.close();
    if (data.getBlobStore() instanceof LocalFsBlobStore) {
        NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
        int versionForKey = BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, data.getConf());
        data.getStormClusterState().setup_blobstore(confKey, nimbusInfo, versionForKey);
    }
}

Example 9 with LocalFsBlobStore

use of com.alibaba.jstorm.blobstore.LocalFsBlobStore in project jstorm by alibaba.

the class NimbusUtils method cleanupCorruptTopologies.

/**
     * clean the topology which is in ZK but not in local dir
     *
     * @throws Exception
     */
public static void cleanupCorruptTopologies(NimbusData data) throws Exception {
    StormClusterState stormClusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    // we have only topology relative files , so we don't need filter
    Set<String> code_ids = Sets.newHashSet(BlobStoreUtils.code_ids(blobStore.listKeys()));
    // get topology in ZK /storms
    Set<String> active_ids = Sets.newHashSet(data.getStormClusterState().active_storms());
    //get topology in zk by blobs
    Set<String> blobsIdsOnZk = Sets.newHashSet(data.getStormClusterState().blobstore(null));
    Set<String> topologyIdsOnZkbyBlobs = BlobStoreUtils.code_ids(blobsIdsOnZk.iterator());
    Set<String> corrupt_ids = Sets.difference(active_ids, code_ids);
    Set<String> redundantIds = Sets.difference(topologyIdsOnZkbyBlobs, code_ids);
    Set<String> unionIds = Sets.union(corrupt_ids, redundantIds);
    // clean the topology which is in ZK but not in local dir
    for (String corrupt : unionIds) {
        LOG.info("Corrupt topology {} has state on zookeeper but doesn't have a local dir on Nimbus. Cleaning up...", corrupt);
        stormClusterState.remove_storm(corrupt);
        if (blobStore instanceof LocalFsBlobStore) {
            List<String> blobKeys = BlobStoreUtils.getKeyListFromId(data, corrupt);
            for (String key : blobKeys) {
                stormClusterState.remove_blobstore_key(key);
                stormClusterState.remove_key_version(key);
            }
        }
    }
    LOG.info("Successfully cleanup all old toplogies");
}

Example 10 with LocalFsBlobStore

use of com.alibaba.jstorm.blobstore.LocalFsBlobStore in project jstorm by alibaba.

the class ServiceHandler method deleteBlob.

@Override
public void deleteBlob(String key) throws TException {
    BlobStore blobStore = data.getBlobStore();
    blobStore.deleteBlob(key);
    if (blobStore instanceof LocalFsBlobStore) {
        try {
            data.getStormClusterState().remove_blobstore_key(key);
            data.getStormClusterState().remove_key_version(key);
        } catch (Exception e) {
            throw new TException(e);
        }
    }
    LOG.info("Deleted blob for key {}", key);
}