Examples with BlobStore com.alibaba.jstorm.blobstore.BlobStore used on opensource projects

Example 1 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class ServiceHandler method createStateInZookeeper.

@Override
public void createStateInZookeeper(String key) throws TException {
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    if (blobStore instanceof LocalFsBlobStore) {
        int versionForKey = BlobStoreUtils.getVersionForKey(key, nimbusInfo, data.getConf());
        try {
            data.getStormClusterState().setup_blobstore(key, nimbusInfo, versionForKey);
        } catch (Exception e) {
            throw new TException("create state in zookeeper error", e);
        }
    }
    LOG.debug("Created state in zookeeper for key:{} for nimbus:{}", key, nimbusInfo);
}

Example 2 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class ServiceHandler method setupStormCode.

/**
     * create local topology files in blobstore and sync metadata to zk
     */
private void setupStormCode(Map<Object, Object> conf, String topologyId, String tmpJarLocation, Map<Object, Object> stormConf, StormTopology topology) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    String codeKey = StormConfig.master_stormcode_key(topologyId);
    String confKey = StormConfig.master_stormconf_key(topologyId);
    // in local mode there is no jar
    if (tmpJarLocation != null) {
        setupJar(tmpJarLocation, topologyId, blobStore, clusterState, nimbusInfo, false);
    }
    blobStore.createBlob(confKey, Utils.serialize(stormConf), new SettableBlobMeta());
    blobStore.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta());
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        clusterState.setup_blobstore(codeKey, nimbusInfo, BlobStoreUtils.getVersionForKey(codeKey, nimbusInfo, conf));
    }
    LOG.info("Successfully create blobstore for topology {}", topologyId);
}

Example 3 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class ServiceHandler method updateTopology.

@Override
public void updateTopology(String name, String uploadedLocation, String updateConf) throws NotAliveException, InvalidTopologyException, TException {
    try {
        //firstly update jar and conf
        checkTopologyActive(data, name, true);
        String topologyId = null;
        StormClusterState stormClusterState = data.getStormClusterState();
        topologyId = Cluster.get_topology_id(stormClusterState, name);
        if (topologyId == null) {
            throw new NotAliveException(name);
        }
        BlobStore blobStore = data.getBlobStore();
        StormClusterState clusterState = data.getStormClusterState();
        NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
        if (uploadedLocation != null) {
            setupJar(uploadedLocation, topologyId, blobStore, clusterState, nimbusInfo, true);
        }
        Map topoConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
        Map<Object, Object> config = (Map<Object, Object>) JStormUtils.from_json(updateConf);
        topoConf.putAll(config);
        String confKey = StormConfig.master_stormconf_key(topologyId);
        BlobStoreUtils.updateBlob(blobStore, confKey, Utils.serialize(topoConf));
        if (blobStore instanceof LocalFsBlobStore) {
            clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        }
        NimbusUtils.transitionName(data, name, true, StatusType.update_topology, config);
        LOG.info("update topology " + name + " successfully");
        notifyTopologyActionListener(name, "updateTopology");
    } catch (NotAliveException e) {
        String errMsg = "Error, no this topology " + name;
        LOG.error(errMsg, e);
        throw new NotAliveException(errMsg);
    } catch (Exception e) {
        String errMsg = "Failed to update topology " + name;
        LOG.error(errMsg, e);
        throw new TException(errMsg);
    }
}

Example 4 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class FollowerRunnable method blobSync.

private synchronized void blobSync() {
    if (!data.isLeader()) {
        try {
            BlobStore blobStore = data.getBlobStore();
            StormClusterState clusterState = data.getStormClusterState();
            Set<String> localKeys = Sets.newHashSet(blobStore.listKeys());
            Set<String> zkKeys = Sets.newHashSet(clusterState.blobstore(blobSyncCallback));
            BlobSynchronizer blobSynchronizer = new BlobSynchronizer(blobStore, data.getConf());
            blobSynchronizer.setNimbusInfo(data.getNimbusHostPortInfo());
            blobSynchronizer.setBlobStoreKeySet(localKeys);
            blobSynchronizer.setZookeeperKeySet(zkKeys);
            blobSynchronizer.syncBlobs();
        } catch (Exception e) {
            LOG.error("blob sync error", e);
        }
    }
}

Example 5 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class FollowerRunnable method setupBlobstore.

// sets up blobstore state for all current keys
private void setupBlobstore() throws Exception {
    BlobStore blobStore = data.getBlobStore();
    StormClusterState clusterState = data.getStormClusterState();
    Set<String> localSetOfKeys = Sets.newHashSet(blobStore.listKeys());
    Set<String> allKeys = Sets.newHashSet(clusterState.active_keys());
    Set<String> localAvailableActiveKeys = Sets.intersection(localSetOfKeys, allKeys);
    // keys on local but not on zk, we will delete it
    Set<String> keysToDelete = Sets.difference(localSetOfKeys, allKeys);
    LOG.debug("deleting keys not on the zookeeper {}", keysToDelete);
    for (String key : keysToDelete) {
        blobStore.deleteBlob(key);
    }
    //    (log-debug "Creating list of key entries for blobstore inside zookeeper" all-keys "local" locally-available-active-keys)
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", allKeys, localAvailableActiveKeys);
    for (String key : localAvailableActiveKeys) {
        int versionForKey = BlobStoreUtils.getVersionForKey(key, data.getNimbusHostPortInfo(), data.getConf());
        clusterState.setup_blobstore(key, data.getNimbusHostPortInfo(), versionForKey);
    }
}