Examples with KeyNotFoundException org.apache.storm.generated.KeyNotFoundException used on opensource projects

Example 16 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class BlobStoreUtils method downloadMissingBlob.

// Download missing blobs from potential nimbodes
public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    ReadableBlobMeta rbm;
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        LOG.debug("Download blob key: {}, NimbusInfo {}", key, nimbusInfo);
        try (NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
            rbm = client.getClient().getBlobMeta(key);
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            in = remoteBlobStore.getBlob(key);
            blobStore.createBlob(key, in, rbm.get_settable(), getNimbusSubject());
            // if key already exists while creating the blob else update it
            Iterator<String> keyIterator = blobStore.listKeys();
            while (keyIterator.hasNext()) {
                if (keyIterator.next().equals(key)) {
                    LOG.debug("Success creating key, {}", key);
                    isSuccess = true;
                    break;
                }
            }
        } catch (IOException | AuthorizationException exception) {
            throw new RuntimeException(exception);
        } catch (KeyAlreadyExistsException kae) {
            LOG.info("KeyAlreadyExistsException Key: {} {}", key, kae);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException Key: {} {}", key, knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception {}", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not download the blob with key: {}", key);
    }
    return isSuccess;
}

Example 17 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class BlobStoreUtils method downloadUpdatedBlob.

// Download updated blobs from potential nimbodes
public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos) throws TTransportException {
    ClientBlobStore remoteBlobStore;
    InputStreamWithMeta in;
    AtomicOutputStream out;
    boolean isSuccess = false;
    LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
    for (NimbusInfo nimbusInfo : nimbusInfos) {
        if (isSuccess) {
            break;
        }
        try (NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
            remoteBlobStore = new NimbusBlobStore();
            remoteBlobStore.setClient(conf, client);
            in = remoteBlobStore.getBlob(key);
            out = blobStore.updateBlob(key, getNimbusSubject());
            byte[] buffer = new byte[2048];
            int len = 0;
            while ((len = in.read(buffer)) > 0) {
                out.write(buffer, 0, len);
            }
            if (out != null) {
                out.close();
            }
            isSuccess = true;
        } catch (IOException | AuthorizationException exception) {
            throw new RuntimeException(exception);
        } catch (KeyNotFoundException knf) {
            // Catching and logging KeyNotFoundException because, if
            // there is a subsequent update and delete, the non-leader
            // nimbodes might throw an exception.
            LOG.info("KeyNotFoundException {}", knf);
        } catch (Exception exp) {
            // Logging an exception while client is connecting
            LOG.error("Exception {}", exp);
        }
    }
    if (!isSuccess) {
        LOG.error("Could not update the blob with key: {}", key);
    }
    return isSuccess;
}

Example 18 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class BlobStoreUtils method updateKeyForBlobStore.

public static void updateKeyForBlobStore(Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
    try {
        // nimbus ha.
        if (nimbusDetails == null) {
            return;
        }
        boolean isListContainsCurrentNimbusInfo = false;
        List<String> stateInfo;
        if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
            return;
        }
        stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
        LOG.debug("StateInfo for update {}", stateInfo);
        Set<NimbusInfo> nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
        for (NimbusInfo nimbusInfo : nimbusInfoList) {
            if (nimbusInfo.getHost().equals(nimbusDetails.getHost())) {
                isListContainsCurrentNimbusInfo = true;
                break;
            }
        }
        if (!isListContainsCurrentNimbusInfo && downloadUpdatedBlob(conf, blobStore, key, nimbusInfoList)) {
            LOG.debug("Updating state inside zookeeper for an update");
            createStateInZookeeper(conf, key, nimbusDetails);
        }
    } catch (NoNodeException | KeyNotFoundException e) {
        //race condition with a delete
        return;
    } catch (Exception exp) {
        throw new RuntimeException(exp);
    }
}

Example 19 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class KeySequenceNumber method getKeySequenceNumber.

public synchronized int getKeySequenceNumber(Map conf) throws KeyNotFoundException {
    TreeSet<Integer> sequenceNumbers = new TreeSet<Integer>();
    CuratorFramework zkClient = BlobStoreUtils.createZKClient(conf);
    try {
        // Key has not been created yet and it is the first time it is being created
        if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
            zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key);
            zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array());
            return INITIAL_SEQUENCE_NUMBER;
        }
        // When all nimbodes go down and one or few of them come up
        // Unfortunately there might not be an exact way to know which one contains the most updated blob,
        // if all go down which is unlikely. Hence there might be a need to update the blob if all go down.
        List<String> stateInfoList = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
        LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), stateInfoList);
        if (stateInfoList.isEmpty()) {
            return getMaxSequenceNumber(zkClient);
        }
        LOG.debug("stateInfoSize {}", stateInfoList.size());
        // if not assign the highest sequence number.
        for (String stateInfo : stateInfoList) {
            sequenceNumbers.add(Integer.parseInt(BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo(stateInfo).getSequenceNumber()));
        }
        // Update scenario 2 and 3 explain the code logic written here
        // especially when nimbus crashes and comes up after and before update
        // respectively.
        int currentSeqNumber = getMaxSequenceNumber(zkClient);
        if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo.isLeader()) {
            if (sequenceNumbers.last() < currentSeqNumber) {
                return currentSeqNumber;
            } else {
                return INITIAL_SEQUENCE_NUMBER - 1;
            }
        }
        // after which nimbus-1 comes back up and a read or update is performed.
        if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo.isLeader()) {
            incrementMaxSequenceNumber(zkClient, currentSeqNumber);
            return currentSeqNumber + 1;
        }
        // Other scenario it covers is when max-seq-number and nimbus seq number are equal.
        if (sequenceNumbers.size() == 1) {
            if (sequenceNumbers.first() < currentSeqNumber) {
                incrementMaxSequenceNumber(zkClient, currentSeqNumber);
                return currentSeqNumber + 1;
            } else {
                incrementMaxSequenceNumber(zkClient, currentSeqNumber);
                return sequenceNumbers.first() + 1;
            }
        }
        // Normal create update sync scenario returns the greatest sequence number in the set
        return sequenceNumbers.last();
    } catch (KeeperException.NoNodeException e) {
        // this should be thrown to the caller to indicate that the key is invalid now
        throw new KeyNotFoundException(key);
    } catch (Exception e) {
        // in other case, just set this to 0 to trigger re-sync later
        LOG.error("Exception {}", e);
        return INITIAL_SEQUENCE_NUMBER - 1;
    } finally {
        if (zkClient != null) {
            zkClient.close();
        }
    }
}

Example 20 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class LocalFsBlobStore method getStoredBlobMeta.

private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundException {
    InputStream in = null;
    try {
        LocalFsBlobStoreFile pf = fbs.read(META_PREFIX + key);
        try {
            in = pf.getInputStream();
        } catch (FileNotFoundException fnf) {
            throw new KeyNotFoundException(key);
        }
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        byte[] buffer = new byte[2048];
        int len;
        while ((len = in.read(buffer)) > 0) {
            out.write(buffer, 0, len);
        }
        in.close();
        in = null;
        return Utils.thriftDeserialize(SettableBlobMeta.class, out.toByteArray());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException e) {
            //Ignored
            }
        }
    }
}