Examples with KeyNotFoundException org.apache.storm.generated.KeyNotFoundException used on opensource projects

Example 1 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class Localizer method updateBlobs.

/**
   * This function updates blobs on the supervisor. It uses a separate thread pool and runs
   * asynchronously of the download and delete.
   */
public List<LocalizedResource> updateBlobs(List<LocalResource> localResources, String user) throws AuthorizationException, KeyNotFoundException, IOException {
    LocalizedResourceSet lrsrcSet = _userRsrc.get(user);
    ArrayList<LocalizedResource> results = new ArrayList<>();
    ArrayList<Callable<LocalizedResource>> updates = new ArrayList<>();
    if (lrsrcSet == null) {
        // resource set must have been removed
        return results;
    }
    ClientBlobStore blobstore = null;
    try {
        blobstore = getClientBlobStore();
        for (LocalResource localResource : localResources) {
            String key = localResource.getBlobName();
            LocalizedResource lrsrc = lrsrcSet.get(key, localResource.shouldUncompress());
            if (lrsrc == null) {
                LOG.warn("blob requested for update doesn't exist: {}", key);
                continue;
            } else {
                // update it if either the version isn't the latest or if any local blob files are missing
                if (!isLocalizedResourceUpToDate(lrsrc, blobstore) || !isLocalizedResourceDownloaded(lrsrc)) {
                    LOG.debug("updating blob: {}", key);
                    updates.add(new DownloadBlob(this, _conf, key, new File(lrsrc.getFilePath()), user, lrsrc.isUncompressed(), true));
                }
            }
        }
    } finally {
        if (blobstore != null) {
            blobstore.shutdown();
        }
    }
    try {
        List<Future<LocalizedResource>> futures = _updateExecService.invokeAll(updates);
        for (Future<LocalizedResource> futureRsrc : futures) {
            try {
                LocalizedResource lrsrc = futureRsrc.get();
                // put the resource just in case it was removed at same time by the cleaner
                LocalizedResourceSet newSet = new LocalizedResourceSet(user);
                LocalizedResourceSet newlrsrcSet = _userRsrc.putIfAbsent(user, newSet);
                if (newlrsrcSet == null) {
                    newlrsrcSet = newSet;
                }
                newlrsrcSet.putIfAbsent(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
                results.add(lrsrc);
            } catch (ExecutionException e) {
                LOG.error("Error updating blob: ", e);
                if (e.getCause() instanceof AuthorizationException) {
                    throw (AuthorizationException) e.getCause();
                }
                if (e.getCause() instanceof KeyNotFoundException) {
                    throw (KeyNotFoundException) e.getCause();
                }
            }
        }
    } catch (RejectedExecutionException re) {
        LOG.error("Error updating blobs : ", re);
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted Exception", ie);
    }
    return results;
}

Example 2 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class Nimbus method setupBlobstore.

/**
     * Sets up blobstore state for all current keys.
     * @throws KeyNotFoundException 
     * @throws AuthorizationException 
     */
private void setupBlobstore() throws AuthorizationException, KeyNotFoundException {
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Set<String> localKeys = new HashSet<>();
    for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
        localKeys.add(it.next());
    }
    Set<String> activeKeys = new HashSet<>(state.activeKeys());
    Set<String> activeLocalKeys = new HashSet<>(localKeys);
    activeLocalKeys.retainAll(activeKeys);
    Set<String> keysToDelete = new HashSet<>(localKeys);
    keysToDelete.removeAll(activeKeys);
    NimbusInfo nimbusInfo = nimbusHostPortInfo;
    LOG.debug("Deleting keys not on the zookeeper {}", keysToDelete);
    for (String toDelete : keysToDelete) {
        store.deleteBlob(toDelete, NIMBUS_SUBJECT);
    }
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", activeKeys, activeLocalKeys);
    for (String key : activeLocalKeys) {
        try {
            state.setupBlobstore(key, nimbusInfo, getVersionForKey(key, nimbusInfo, conf));
        } catch (KeyNotFoundException e) {
            // invalid key, remove it from blobstore
            store.deleteBlob(key, NIMBUS_SUBJECT);
        }
    }
}

Example 3 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class UpdateBlobs method updateBlobsForTopology.

/**
     * Update each blob listed in the topology configuration if the latest version of the blob has not been downloaded.
     * 
     * @param conf
     * @param stormId
     * @param localizer
     * @throws IOException
     */
private void updateBlobsForTopology(Map conf, String stormId, Localizer localizer) throws IOException {
    Map stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId);
    Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
    String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
    List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
    try {
        localizer.updateBlobs(localresources, user);
    } catch (AuthorizationException authExp) {
        LOG.error("AuthorizationException error", authExp);
    } catch (KeyNotFoundException knf) {
        LOG.error("KeyNotFoundException error", knf);
    }
}

Example 4 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class KeySequenceNumber method getKeySequenceNumber.

public synchronized int getKeySequenceNumber(CuratorFramework zkClient) throws KeyNotFoundException {
    TreeSet<Integer> sequenceNumbers = new TreeSet<Integer>();
    try {
        // Key has not been created yet and it is the first time it is being created
        if (zkClient.checkExists().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key) == null) {
            zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key);
            zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array());
            return INITIAL_SEQUENCE_NUMBER;
        }
        // When all nimbodes go down and one or few of them come up
        // Unfortunately there might not be an exact way to know which one contains the most updated blob,
        // if all go down which is unlikely. Hence there might be a need to update the blob if all go down.
        List<String> stateInfoList = zkClient.getChildren().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key);
        LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), stateInfoList);
        if (stateInfoList.isEmpty()) {
            return getMaxSequenceNumber(zkClient);
        }
        LOG.debug("stateInfoSize {}", stateInfoList.size());
        // if not assign the highest sequence number.
        for (String stateInfo : stateInfoList) {
            sequenceNumbers.add(Integer.parseInt(BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo(stateInfo).getSequenceNumber()));
        }
        // Update scenario 2 and 3 explain the code logic written here
        // especially when nimbus crashes and comes up after and before update
        // respectively.
        int currentSeqNumber = getMaxSequenceNumber(zkClient);
        if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo.isLeader()) {
            if (sequenceNumbers.last() < currentSeqNumber) {
                return currentSeqNumber;
            } else {
                return INITIAL_SEQUENCE_NUMBER - 1;
            }
        }
        // after which nimbus-1 comes back up and a read or update is performed.
        if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo.isLeader()) {
            incrementMaxSequenceNumber(zkClient, currentSeqNumber);
            return currentSeqNumber + 1;
        }
        // Other scenario it covers is when max-seq-number and nimbus seq number are equal.
        if (sequenceNumbers.size() == 1) {
            if (sequenceNumbers.first() < currentSeqNumber) {
                incrementMaxSequenceNumber(zkClient, currentSeqNumber);
                return currentSeqNumber + 1;
            } else {
                incrementMaxSequenceNumber(zkClient, currentSeqNumber);
                return sequenceNumbers.first() + 1;
            }
        }
        // Normal create update sync scenario returns the greatest sequence number in the set
        return sequenceNumbers.last();
    } catch (KeeperException.NoNodeException e) {
        // this should be thrown to the caller to indicate that the key is invalid now
        throw new WrappedKeyNotFoundException(key);
    } catch (Exception e) {
        // in other case, just set this to 0 to trigger re-sync later
        LOG.error("Exception {}", e);
        return INITIAL_SEQUENCE_NUMBER - 1;
    }
}

Example 5 with KeyNotFoundException

use of org.apache.storm.generated.KeyNotFoundException in project storm by apache.

the class LocalFsBlobStore method startSyncBlobs.

@Override
public void startSyncBlobs() throws KeyNotFoundException, AuthorizationException {
    // register call back for blob-store
    this.stormClusterState.blobstore(() -> {
        try {
            blobSync();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
    setupBlobstore();
    // Schedule nimbus code sync thread to sync code from other nimbuses.
    this.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            try {
                blobSync();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    }, 0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CODE_SYNC_FREQ_SECS)) * 1000);
}