Examples with BlobStore org.apache.storm.blobstore.BlobStore used on opensource projects

Example 6 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method blobSync.

private void blobSync() throws Exception {
    if ("distributed".equals(conf.get(Config.STORM_CLUSTER_MODE))) {
        if (!isLeader()) {
            IStormClusterState state = stormClusterState;
            NimbusInfo nimbusInfo = nimbusHostPortInfo;
            BlobStore store = blobStore;
            Set<String> allKeys = new HashSet<>();
            for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
                allKeys.add(it.next());
            }
            Set<String> zkKeys = new HashSet<>(state.blobstore(() -> {
                try {
                    this.blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }));
            LOG.debug("blob-sync blob-store-keys {} zookeeper-keys {}", allKeys, zkKeys);
            BlobSynchronizer sync = new BlobSynchronizer(store, conf);
            sync.setNimbusInfo(nimbusInfo);
            sync.setBlobStoreKeySet(allKeys);
            sync.setZookeeperKeySet(zkKeys);
            sync.syncBlobs();
        }
    //else not leader (NOOP)
    }
//else local (NOOP)
}

Example 7 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method createStateInZookeeper.

@Override
public void createStateInZookeeper(String key) throws TException {
    try {
        IStormClusterState state = stormClusterState;
        BlobStore store = blobStore;
        NimbusInfo ni = nimbusHostPortInfo;
        if (store instanceof LocalFsBlobStore) {
            state.setupBlobstore(key, ni, getVersionForKey(key, ni, conf));
        }
        LOG.debug("Created state in zookeeper {} {} {}", state, store, ni);
    } catch (Exception e) {
        LOG.warn("Exception while creating state in zookeeper - key: " + key, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 8 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class BlobStoreTest method testHdfsReplication.

@Test
public void testHdfsReplication() throws Exception {
    BlobStore store = initHdfs("/storm/blobstoreReplication");
    testReplication("/storm/blobstoreReplication/test", store);
}

Example 9 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method computeExecutors.

private List<List<Integer>> computeExecutors(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    BlobStore store = blobStore;
    assert (base != null);
    Map<String, Integer> compToExecutors = base.get_component_executors();
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
    StormTopology topology = readStormTopologyAsNimbus(topoId, store);
    List<List<Integer>> ret = new ArrayList<>();
    if (compToExecutors != null) {
        Map<Integer, String> taskInfo = StormCommon.stormTaskInfo(topology, topoConf);
        Map<String, List<Integer>> compToTaskList = Utils.reverseMap(taskInfo);
        for (Entry<String, List<Integer>> entry : compToTaskList.entrySet()) {
            List<Integer> comps = entry.getValue();
            comps.sort(null);
            Integer numExecutors = compToExecutors.get(entry.getKey());
            if (numExecutors != null) {
                List<List<Integer>> partitioned = Utils.partitionFixed(numExecutors, comps);
                for (List<Integer> partition : partitioned) {
                    ret.add(Arrays.asList(partition.get(0), partition.get(partition.size() - 1)));
                }
            }
        }
    }
    return ret;
}

Example 10 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method startTopology.

private void startTopology(String topoName, String topoId, TopologyStatus initStatus) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    assert (TopologyStatus.ACTIVE == initStatus || TopologyStatus.INACTIVE == initStatus);
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Map<String, Object> topoConf = readTopoConf(topoId, store);
    StormTopology topology = StormCommon.systemTopology(topoConf, readStormTopology(topoId, store));
    Map<String, Integer> numExecutors = new HashMap<>();
    for (Entry<String, Object> entry : StormCommon.allComponents(topology).entrySet()) {
        numExecutors.put(entry.getKey(), StormCommon.numStartExecutors(entry.getValue()));
    }
    LOG.info("Activating {}: {}", topoName, topoId);
    StormBase base = new StormBase();
    base.set_name(topoName);
    base.set_launch_time_secs(Time.currentTimeSecs());
    base.set_status(initStatus);
    base.set_num_workers(Utils.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 0));
    base.set_component_executors(numExecutors);
    base.set_owner((String) topoConf.get(Config.TOPOLOGY_SUBMITTER_USER));
    base.set_component_debug(new HashMap<>());
    state.activateStorm(topoId, base);
    notifyTopologyActionListener(topoName, "activate");
}