Examples with NimbusInfo backtype.storm.nimbus.NimbusInfo used on opensource projects

Example 1 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class LocalFsBlobStore method checkForBlobOrDownload.

//This additional check and download is for nimbus high availability in case you have more than one nimbus
public boolean checkForBlobOrDownload(String key) {
    boolean checkBlobDownload = false;
    long start = System.currentTimeMillis();
    ReentrantLock lock = getLockForKey(key);
    try {
        lock.lock();
        long getKeyStart = System.currentTimeMillis();
        List<String> keyList = BlobStoreUtils.getKeyListFromBlobStore(this);
        LOG.info("list blob keys, size:{}, cost:{}", keyList.size(), System.currentTimeMillis() - getKeyStart);
        if (!keyList.contains(key)) {
            if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + key) != null) {
                Set<NimbusInfo> nimbusSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
                Set<NimbusInfo> filterNimbusSet = new HashSet<>();
                for (NimbusInfo nimbusInfo : nimbusSet) {
                    if (!nimbusInfo.getHostPort().equals(this.nimbusInfo.getHostPort())) {
                        filterNimbusSet.add(nimbusInfo);
                    }
                }
                if (BlobStoreUtils.downloadMissingBlob(conf, this, key, filterNimbusSet)) {
                    LOG.debug("Updating blobs state");
                    BlobStoreUtils.createStateInZookeeper(conf, key, nimbusInfo);
                    checkBlobDownload = true;
                }
            }
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        unlockForKey(key, lock);
    }
    LOG.debug("checkForBlobOrDownload, key:{}, cost:{}", key, System.currentTimeMillis() - start);
    return checkBlobDownload;
}

Example 2 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class BlobStoreUtils method downloadResourcesAsSupervisor.

/**
     * Meant to be called only by the supervisor for stormjar/stormconf/stormcode files.
     *
     * @param key
     * @param localFile
     * @param cb
     * @throws KeyNotFoundException
     * @throws IOException
     */
public static void downloadResourcesAsSupervisor(String key, String localFile, ClientBlobStore cb, Map conf) throws KeyNotFoundException, IOException {
    if (cb instanceof NimbusBlobStore) {
        List<NimbusInfo> nimbusInfos = null;
        CuratorFramework zkClient = null;
        try {
            zkClient = BlobStoreUtils.createZKClient(conf);
            nimbusInfos = Lists.newArrayList(BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key));
            Collections.shuffle(nimbusInfos);
        } catch (Exception e) {
            LOG.error("get available nimbus for blob key:{} error", e);
            return;
        } finally {
            if (zkClient != null) {
                zkClient.close();
                zkClient = null;
            }
        }
        if (nimbusInfos != null) {
            for (NimbusInfo nimbusInfo : nimbusInfos) {
                try {
                    NimbusClient nimbusClient = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort());
                    cb.setClient(conf, nimbusClient);
                } catch (TTransportException e) {
                    // ignore
                    continue;
                }
                LOG.info("download blob {} from nimbus {}:{}", key, nimbusInfo.getHost(), nimbusInfo.getPort());
                downloadResourcesAsSupervisorDirect(key, localFile, cb);
            }
        }
    } else {
        downloadResourcesAsSupervisorDirect(key, localFile, cb);
    }
}

Example 3 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class ServiceHandler method createStateInZookeeper.

@Override
public void createStateInZookeeper(String key) throws TException {
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    if (blobStore instanceof LocalFsBlobStore) {
        int versionForKey = BlobStoreUtils.getVersionForKey(key, nimbusInfo, data.getConf());
        try {
            data.getStormClusterState().setup_blobstore(key, nimbusInfo, versionForKey);
        } catch (Exception e) {
            throw new TException("create state in zookeeper error", e);
        }
    }
    LOG.debug("Created state in zookeeper for key:{} for nimbus:{}", key, nimbusInfo);
}

Example 4 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class ServiceHandler method setupStormCode.

/**
     * create local topology files in blobstore and sync metadata to zk
     */
private void setupStormCode(Map<Object, Object> conf, String topologyId, String tmpJarLocation, Map<Object, Object> stormConf, StormTopology topology) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    String codeKey = StormConfig.master_stormcode_key(topologyId);
    String confKey = StormConfig.master_stormconf_key(topologyId);
    // in local mode there is no jar
    if (tmpJarLocation != null) {
        setupJar(tmpJarLocation, topologyId, blobStore, clusterState, nimbusInfo, false);
    }
    blobStore.createBlob(confKey, Utils.serialize(stormConf), new SettableBlobMeta());
    blobStore.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta());
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        clusterState.setup_blobstore(codeKey, nimbusInfo, BlobStoreUtils.getVersionForKey(codeKey, nimbusInfo, conf));
    }
    LOG.info("Successfully create blobstore for topology {}", topologyId);
}

Example 5 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class ServiceHandler method updateTopology.

@Override
public void updateTopology(String name, String uploadedLocation, String updateConf) throws NotAliveException, InvalidTopologyException, TException {
    try {
        //firstly update jar and conf
        checkTopologyActive(data, name, true);
        String topologyId = null;
        StormClusterState stormClusterState = data.getStormClusterState();
        topologyId = Cluster.get_topology_id(stormClusterState, name);
        if (topologyId == null) {
            throw new NotAliveException(name);
        }
        BlobStore blobStore = data.getBlobStore();
        StormClusterState clusterState = data.getStormClusterState();
        NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
        if (uploadedLocation != null) {
            setupJar(uploadedLocation, topologyId, blobStore, clusterState, nimbusInfo, true);
        }
        Map topoConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
        Map<Object, Object> config = (Map<Object, Object>) JStormUtils.from_json(updateConf);
        topoConf.putAll(config);
        String confKey = StormConfig.master_stormconf_key(topologyId);
        BlobStoreUtils.updateBlob(blobStore, confKey, Utils.serialize(topoConf));
        if (blobStore instanceof LocalFsBlobStore) {
            clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        }
        NimbusUtils.transitionName(data, name, true, StatusType.update_topology, config);
        LOG.info("update topology " + name + " successfully");
        notifyTopologyActionListener(name, "updateTopology");
    } catch (NotAliveException e) {
        String errMsg = "Error, no this topology " + name;
        LOG.error(errMsg, e);
        throw new NotAliveException(errMsg);
    } catch (Exception e) {
        String errMsg = "Failed to update topology " + name;
        LOG.error(errMsg, e);
        throw new TException(errMsg);
    }
}