Examples with NimbusInfo org.apache.storm.nimbus.NimbusInfo used on opensource projects

Example 11 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class BlobStoreUtils method downloadMissingBlob.

// Download missing blobs from potential nimbodes
public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    ReadableBlobMeta rbm;
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        LOG.debug("Download blob key: {}, NimbusInfo {}", key, nimbusInfo);
        try (NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
            rbm = client.getClient().getBlobMeta(key);
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            in = remoteBlobStore.getBlob(key);
            blobStore.createBlob(key, in, rbm.get_settable(), getNimbusSubject());
            // if key already exists while creating the blob else update it
            Iterator<String> keyIterator = blobStore.listKeys();
            while (keyIterator.hasNext()) {
                if (keyIterator.next().equals(key)) {
                    LOG.debug("Success creating key, {}", key);
                    isSuccess = true;
                    break;
                }
            }
        } catch (IOException | AuthorizationException exception) {
            throw new RuntimeException(exception);
        } catch (KeyAlreadyExistsException kae) {
            LOG.info("KeyAlreadyExistsException Key: {} {}", key, kae);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException Key: {} {}", key, knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception {}", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not download the blob with key: {}", key);
    }
    return isSuccess;
}

Example 12 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class BlobStoreUtils method downloadUpdatedBlob.

// Download updated blobs from potential nimbodes
public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    AtomicOutputStream out;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        try (NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            in = remoteBlobStore.getBlob(key);
            out = blobStore.updateBlob(key, getNimbusSubject());
            byte[] buffer = new byte[2048];
            int len = 0;
            while ((len = in.read(buffer)) > 0) {
                out.write(buffer, 0, len);
            }
            if (out != null) {
                out.close();
            }
            isSuccess = true;
        } catch (IOException | AuthorizationException exception) {
            throw new RuntimeException(exception);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException {}", knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception {}", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not update the blob with key: {}", key);
    }
    return isSuccess;
}

Example 13 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class BlobStoreUtils method updateKeyForBlobStore.

public static void updateKeyForBlobStore(Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
    try {
        // nimbus ha.
        if (nimbusDetails == null) {
            return;
        }
        boolean isListContainsCurrentNimbusInfo = false;
        List<String> stateInfo;
        if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
            return;
        }
        stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
        LOG.debug("StateInfo for update {}", stateInfo);
        Set<NimbusInfo> nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
        for (NimbusInfo nimbusInfo : nimbusInfoList) {
            if (nimbusInfo.getHost().equals(nimbusDetails.getHost())) {
                isListContainsCurrentNimbusInfo = true;
                break;
            }
        }
        if (!isListContainsCurrentNimbusInfo && downloadUpdatedBlob(conf, blobStore, key, nimbusInfoList)) {
            LOG.debug("Updating state inside zookeeper for an update");
            createStateInZookeeper(conf, key, nimbusDetails);
        }
    } catch (NoNodeException | KeyNotFoundException e) {
        //race condition with a delete
        return;
    } catch (Exception exp) {
        throw new RuntimeException(exp);
    }
}

Example 14 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class BlobSynchronizerTest method testNimbodesWithLatestVersionOfBlob.

@Test
public void testNimbodesWithLatestVersionOfBlob() throws Exception {
    TestingServer server = new TestingServer();
    CuratorFramework zkClient = CuratorFrameworkFactory.newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3));
    zkClient.start();
    // Creating nimbus hosts containing latest version of blob
    zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
    zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
    Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1");
    assertEquals("Failed to get the correct nimbus hosts with latest blob version", (set.iterator().next()).getHost(), "nimbus2");
    zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
    zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
    zkClient.close();
    server.close();
}