Examples with NimbusInfo backtype.storm.nimbus.NimbusInfo used on opensource projects

Example 6 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class StormConfig method write_nimbus_topology_code.

public static void write_nimbus_topology_code(String topologyId, byte[] data, NimbusData nimbusData) throws Exception {
    String codeKey = master_stormcode_key(topologyId);
    AtomicOutputStream out = nimbusData.getBlobStore().updateBlob(codeKey);
    out.write(data);
    out.close();
    if (nimbusData.getBlobStore() instanceof LocalFsBlobStore) {
        NimbusInfo nimbusInfo = nimbusData.getNimbusHostPortInfo();
        int versionForKey = BlobStoreUtils.getVersionForKey(codeKey, nimbusInfo, nimbusData.getConf());
        nimbusData.getStormClusterState().setup_blobstore(codeKey, nimbusInfo, versionForKey);
    }
}

Example 7 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class BlobStoreUtils method downloadUpdatedBlob.

// Download updated blobs from potential nimbodes
public static boolean downloadUpdatedBlob(Map conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    NimbusClient client;
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    AtomicOutputStream out;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        try {
            client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            isSuccess = updateBlob(blobStore, key, remoteBlobStore.getBlob(key));
        } catch (IOException exception) {
            throw new RuntimeException(exception);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException {}", knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception {}", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not update the blob with key" + key);
    }
    return isSuccess;
}

Example 8 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class BlobStoreUtils method downloadMissingBlob.

// Download missing blobs from potential nimbodes
public static boolean downloadMissingBlob(Map conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    NimbusClient client;
    ReadableBlobMeta rbm;
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        try {
            client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
            rbm = client.getClient().getBlobMeta(key);
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            in = remoteBlobStore.getBlob(key);
            blobStore.createBlob(key, in, rbm.get_settable());
            // if key already exists while creating the blob else update it
            Iterator<String> keyIterator = blobStore.listKeys();
            while (keyIterator.hasNext()) {
                if (keyIterator.next().equals(key)) {
                    LOG.debug("Success creating key, {}", key);
                    isSuccess = true;
                    break;
                }
            }
        } catch (IOException exception) {
            throw new RuntimeException(exception);
        } catch (KeyAlreadyExistsException kae) {
            LOG.info("KeyAlreadyExistsException Key: {} {}", key, kae);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException Key: {} {}", key, knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception ", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not download blob with key" + key);
    }
    return isSuccess;
}

Example 9 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class BlobStoreUtils method updateKeyForBlobStore.

public static void updateKeyForBlobStore(Map conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
    try {
        // nimbus ha.
        if (nimbusDetails == null) {
            return;
        }
        boolean isListContainsCurrentNimbusInfo = false;
        List<String> stateInfo;
        if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
            return;
        }
        stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
        LOG.debug("StateInfo for update {}", stateInfo);
        Set<NimbusInfo> nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
        for (NimbusInfo nimbusInfo : nimbusInfoList) {
            if (nimbusInfo.getHostPort().equals(nimbusDetails.getHostPort())) {
                isListContainsCurrentNimbusInfo = true;
                break;
            }
        }
        if (!isListContainsCurrentNimbusInfo && downloadUpdatedBlob(conf, blobStore, key, nimbusInfoList)) {
            LOG.debug("Updating state inside zookeeper for an update");
            createStateInZookeeper(conf, key, nimbusDetails);
        }
    } catch (Exception exp) {
        throw new RuntimeException(exp);
    }
}

Example 10 with NimbusInfo

use of backtype.storm.nimbus.NimbusInfo in project jstorm by alibaba.

the class BlobSynchronizer method syncBlobs.

public synchronized void syncBlobs() {
    CuratorFramework zkClient = null;
    try {
        LOG.debug("Sync blobs - blobstore keys {}, zookeeper keys {}", getBlobStoreKeySet(), getZookeeperKeySet());
        zkClient = BlobStoreUtils.createZKClient(conf);
        // delete useless keys
        deleteKeySetFromBlobStoreNotOnZookeeper(getBlobStoreKeySet(), getZookeeperKeySet());
        // update exist keys
        updateKeySetForBlobStore(getBlobStoreKeySet(), zkClient);
        // download missing keys
        Set<String> keySetToDownload = getKeySetToDownload(getBlobStoreKeySet(), getZookeeperKeySet());
        LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", getBlobStoreKeySet(), getZookeeperKeySet(), keySetToDownload);
        for (String key : keySetToDownload) {
            Set<NimbusInfo> nimbusInfoSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
            if (BlobStoreUtils.downloadMissingBlob(conf, blobStore, key, nimbusInfoSet)) {
                BlobStoreUtils.createStateInZookeeper(conf, key, nimbusInfo);
            }
        }
    } catch (InterruptedException exp) {
        LOG.error("InterruptedException {}", exp);
    } catch (Exception exp) {
        throw new RuntimeException(exp);
    } finally {
        if (zkClient != null) {
            zkClient.close();
        }
    }
}