Examples with BlobStore com.alibaba.jstorm.blobstore.BlobStore used on opensource projects

Example 6 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class NimbusUtils method cleanupCorruptTopologies.

/**
     * clean the topology which is in ZK but not in local dir
     *
     * @throws Exception
     */
public static void cleanupCorruptTopologies(NimbusData data) throws Exception {
    StormClusterState stormClusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    // we have only topology relative files , so we don't need filter
    Set<String> code_ids = Sets.newHashSet(BlobStoreUtils.code_ids(blobStore.listKeys()));
    // get topology in ZK /storms
    Set<String> active_ids = Sets.newHashSet(data.getStormClusterState().active_storms());
    //get topology in zk by blobs
    Set<String> blobsIdsOnZk = Sets.newHashSet(data.getStormClusterState().blobstore(null));
    Set<String> topologyIdsOnZkbyBlobs = BlobStoreUtils.code_ids(blobsIdsOnZk.iterator());
    Set<String> corrupt_ids = Sets.difference(active_ids, code_ids);
    Set<String> redundantIds = Sets.difference(topologyIdsOnZkbyBlobs, code_ids);
    Set<String> unionIds = Sets.union(corrupt_ids, redundantIds);
    // clean the topology which is in ZK but not in local dir
    for (String corrupt : unionIds) {
        LOG.info("Corrupt topology {} has state on zookeeper but doesn't have a local dir on Nimbus. Cleaning up...", corrupt);
        stormClusterState.remove_storm(corrupt);
        if (blobStore instanceof LocalFsBlobStore) {
            List<String> blobKeys = BlobStoreUtils.getKeyListFromId(data, corrupt);
            for (String key : blobKeys) {
                stormClusterState.remove_blobstore_key(key);
                stormClusterState.remove_key_version(key);
            }
        }
    }
    LOG.info("Successfully cleanup all old toplogies");
}

Example 7 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class TopologyNettyMgr method getTopology.

public boolean getTopology(String topologyId) {
    BlobStore blobStore = null;
    try {
        String topologyName = Common.topologyIdToName(topologyId);
        Boolean isEnable = setting.get(topologyName);
        if (isEnable != null) {
            return isEnable;
        }
        blobStore = BlobStoreUtils.getNimbusBlobStore(nimbusConf, NimbusInfo.fromConf(nimbusConf));
        Map topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, blobStore);
        isEnable = getTopology(topologyConf);
        setting.put(topologyName, isEnable);
        LOG.info("{} netty metrics setting is {}", topologyName, isEnable);
        return isEnable;
    } catch (Exception e) {
        LOG.info("Failed to get {} netty metrics setting ", topologyId);
        return true;
    } finally {
        if (blobStore != null) {
            blobStore.shutdown();
            blobStore = null;
        }
    }
}

Example 8 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class SyncSupervisorEvent method downloadLocalStormCode.

private void downloadLocalStormCode(Map conf, String topologyId, String masterCodeDir) throws IOException, TException {
    // STORM_LOCAL_DIR/supervisor/tmp/(UUID)
    String tmproot = StormConfig.supervisorTmpDir(conf) + File.separator + UUID.randomUUID().toString();
    // STORM-LOCAL-DIR/supervisor/stormdist/storm-id
    String stormroot = StormConfig.supervisor_stormdist_root(conf, topologyId);
    BlobStore blobStore = null;
    try {
        blobStore = BlobStoreUtils.getNimbusBlobStore(conf, masterCodeDir, null);
        FileUtils.forceMkdir(new File(tmproot));
        blobStore.readBlobTo(StormConfig.master_stormcode_key(topologyId), new FileOutputStream(StormConfig.stormcode_path(tmproot)));
        blobStore.readBlobTo(StormConfig.master_stormconf_key(topologyId), new FileOutputStream(StormConfig.stormconf_path(tmproot)));
    } finally {
        if (blobStore != null)
            blobStore.shutdown();
    }
    File srcDir = new File(tmproot);
    File destDir = new File(stormroot);
    try {
        FileUtils.moveDirectory(srcDir, destDir);
    } catch (FileExistsException e) {
        FileUtils.copyDirectory(srcDir, destDir);
        FileUtils.deleteQuietly(srcDir);
    }
    ClassLoader classloader = Thread.currentThread().getContextClassLoader();
    String resourcesJar = resourcesJar();
    URL url = classloader.getResource(StormConfig.RESOURCES_SUBDIR);
    String targetDir = stormroot + '/' + StormConfig.RESOURCES_SUBDIR;
    if (resourcesJar != null) {
        LOG.info("Extracting resources from jar at " + resourcesJar + " to " + targetDir);
        // extract dir
        JStormUtils.extractDirFromJar(resourcesJar, StormConfig.RESOURCES_SUBDIR, stormroot);
    // from jar;;
    // util.clj
    } else if (url != null) {
        LOG.info("Copying resources at " + url.toString() + " to " + targetDir);
        FileUtils.copyDirectory(new File(url.getFile()), (new File(targetDir)));
    }
}

Example 9 with BlobStore

use of com.alibaba.jstorm.blobstore.BlobStore in project jstorm by alibaba.

the class ServiceHandler method deleteBlob.

@Override
public void deleteBlob(String key) throws TException {
    BlobStore blobStore = data.getBlobStore();
    blobStore.deleteBlob(key);
    if (blobStore instanceof LocalFsBlobStore) {
        try {
            data.getStormClusterState().remove_blobstore_key(key);
            data.getStormClusterState().remove_key_version(key);
        } catch (Exception e) {
            throw new TException(e);
        }
    }
    LOG.info("Deleted blob for key {}", key);
}