Examples with CacheKey org.apache.hadoop.hdds.utils.db.cache.CacheKey used on opensource projects

Example 1 with CacheKey

use of org.apache.hadoop.hdds.utils.db.cache.CacheKey in project ozone by apache.

the class KeyManagerImpl method listStatus.

/**
 * List the status for a file or a directory and its contents.
 *
 * @param args       Key args
 * @param recursive  For a directory if true all the descendants of a
 *                   particular directory are listed
 * @param startKey   Key from which listing needs to start. If startKey exists
 *                   its status is included in the final list.
 * @param numEntries Number of entries to list from the start key
 * @param clientAddress a hint to key manager, order the datanode in returned
 *                      pipeline by distance between client and datanode.
 * @return list of file status
 */
@Override
@SuppressWarnings("methodlength")
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, String clientAddress) throws IOException {
    Preconditions.checkNotNull(args, "Key args can not be null");
    String volName = args.getVolumeName();
    String buckName = args.getBucketName();
    List<OzoneFileStatus> fileStatusList = new ArrayList<>();
    if (numEntries <= 0) {
        return fileStatusList;
    }
    if (isBucketFSOptimized(volName, buckName)) {
        return listStatusFSO(args, recursive, startKey, numEntries, clientAddress);
    }
    String volumeName = args.getVolumeName();
    String bucketName = args.getBucketName();
    String keyName = args.getKeyName();
    // A map sorted by OmKey to combine results from TableCache and DB.
    TreeMap<String, OzoneFileStatus> cacheKeyMap = new TreeMap<>();
    if (Strings.isNullOrEmpty(startKey)) {
        OzoneFileStatus fileStatus = getFileStatus(args, clientAddress);
        if (fileStatus.isFile()) {
            return Collections.singletonList(fileStatus);
        }
        // keyName is a directory
        startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
    }
    // Note: eliminating the case where startCacheKey could end with '//'
    String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded(metadataManager.getOzoneKey(volumeName, bucketName, keyName));
    metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName);
    Table keyTable = metadataManager.getKeyTable(getBucketLayout(metadataManager, volName, buckName));
    TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator;
    try {
        Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter = keyTable.cacheIterator();
        String startCacheKey = OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName + OZONE_URI_DELIMITER + ((startKey.equals(OZONE_URI_DELIMITER)) ? "" : startKey);
        // First, find key in TableCache
        listStatusFindKeyInTableCache(cacheIter, keyArgs, startCacheKey, recursive, cacheKeyMap);
        iterator = keyTable.iterator();
    } finally {
        metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName);
    }
    // Then, find key in DB
    String seekKeyInDb = metadataManager.getOzoneKey(volumeName, bucketName, startKey);
    Table.KeyValue<String, OmKeyInfo> entry = iterator.seek(seekKeyInDb);
    int countEntries = 0;
    if (iterator.hasNext()) {
        if (entry.getKey().equals(keyArgs)) {
            // Skip the key itself, since we are listing inside the directory
            iterator.next();
        }
        // Iterate through seek results
        while (iterator.hasNext() && numEntries - countEntries > 0) {
            entry = iterator.next();
            String entryInDb = entry.getKey();
            OmKeyInfo omKeyInfo = entry.getValue();
            if (entryInDb.startsWith(keyArgs)) {
                String entryKeyName = omKeyInfo.getKeyName();
                if (recursive) {
                    if (!isKeyDeleted(entryInDb, keyTable)) {
                        cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(entryKeyName)));
                        countEntries++;
                    }
                } else {
                    // get the child of the directory to list from the entry. For
                    // example if directory to list is /a and entry is /a/b/c where
                    // c is a file. The immediate child is b which is a directory. c
                    // should not be listed as child of a.
                    String immediateChild = OzoneFSUtils.getImmediateChild(entryKeyName, keyName);
                    boolean isFile = OzoneFSUtils.isFile(immediateChild);
                    if (isFile) {
                        if (!isKeyDeleted(entryInDb, keyTable)) {
                            cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, scmBlockSize, !isFile));
                            countEntries++;
                        }
                    } else {
                        // if entry is a directory
                        if (!isKeyDeleted(entryInDb, keyTable)) {
                            if (!entryKeyName.equals(immediateChild)) {
                                OmKeyInfo fakeDirEntry = createDirectoryKey(omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), immediateChild, omKeyInfo.getAcls());
                                cacheKeyMap.put(entryInDb, new OzoneFileStatus(fakeDirEntry, scmBlockSize, true));
                            } else {
                                // If entryKeyName matches dir name, we have the info
                                cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, 0, true));
                            }
                            countEntries++;
                        }
                        // skip the other descendants of this child directory.
                        iterator.seek(getNextGreaterString(volumeName, bucketName, immediateChild));
                    }
                }
            } else {
                break;
            }
        }
    }
    countEntries = 0;
    // Convert results in cacheKeyMap to List
    for (OzoneFileStatus fileStatus : cacheKeyMap.values()) {
        // No need to check if a key is deleted or not here, this is handled
        // when adding entries to cacheKeyMap from DB.
        fileStatusList.add(fileStatus);
        countEntries++;
        if (countEntries >= numEntries) {
            break;
        }
    }
    // Clean up temp map and set
    cacheKeyMap.clear();
    List<OmKeyInfo> keyInfoList = new ArrayList<>(fileStatusList.size());
    fileStatusList.stream().map(s -> s.getKeyInfo()).forEach(keyInfoList::add);
    if (args.getLatestVersionLocation()) {
        slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
    }
    refreshPipeline(keyInfoList);
    if (args.getSortDatanodes()) {
        sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0]));
    }
    return fileStatusList;
}

Example 2 with CacheKey

use of org.apache.hadoop.hdds.utils.db.cache.CacheKey in project ozone by apache.

the class KeyManagerImpl method listStatusFindFilesInTableCache.

/**
 * Helper function for listStatus to find key in FileTableCache.
 */
@SuppressWarnings("parameternumber")
private int listStatusFindFilesInTableCache(Map<String, OzoneFileStatus> cacheKeyMap, Table<String, OmKeyInfo> keyTable, long prefixKeyInDB, String seekKeyInDB, String prefixKeyPath, String startKey, int countEntries, long numEntries, Set<String> deletedKeySet) {
    Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter = keyTable.cacheIterator();
    // TODO: recursive list will be handled in HDDS-4360 jira.
    while (cacheIter.hasNext() && numEntries - countEntries > 0) {
        Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry = cacheIter.next();
        String cacheKey = entry.getKey().getCacheKey();
        OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
        // cacheOmKeyInfo is null if an entry is deleted in cache
        if (cacheOmKeyInfo == null) {
            deletedKeySet.add(cacheKey);
            continue;
        }
        // make OmKeyInfo local copy to reset keyname to "fullKeyPath".
        // In DB keyName stores only the leaf node but the list
        // returning to the user should have full path.
        OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject();
        omKeyInfo.setFileName(omKeyInfo.getKeyName());
        String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, omKeyInfo.getKeyName());
        omKeyInfo.setKeyName(fullKeyPath);
        countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB, seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo, false);
    }
    return countEntries;
}

Example 3 with CacheKey

use of org.apache.hadoop.hdds.utils.db.cache.CacheKey in project ozone by apache.

the class KeyManagerImpl method listStatusFindKeyInTableCache.

/**
 * Helper function for listStatus to find key in TableCache.
 */
private void listStatusFindKeyInTableCache(Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter, String keyArgs, String startCacheKey, boolean recursive, TreeMap<String, OzoneFileStatus> cacheKeyMap) {
    while (cacheIter.hasNext()) {
        Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry = cacheIter.next();
        String cacheKey = entry.getKey().getCacheKey();
        if (cacheKey.equals(keyArgs)) {
            continue;
        }
        OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
        // cacheOmKeyInfo is null if an entry is deleted in cache
        if (cacheOmKeyInfo != null && cacheKey.startsWith(startCacheKey) && cacheKey.compareTo(startCacheKey) >= 0) {
            if (!recursive) {
                String remainingKey = StringUtils.stripEnd(cacheKey.substring(startCacheKey.length()), OZONE_URI_DELIMITER);
                // For non-recursive, the remaining part of key can't have '/'
                if (remainingKey.contains(OZONE_URI_DELIMITER)) {
                    continue;
                }
            }
            OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey));
            cacheKeyMap.put(cacheKey, fileStatus);
        }
    }
}

Example 4 with CacheKey

use of org.apache.hadoop.hdds.utils.db.cache.CacheKey in project ozone by apache.

the class OmMetadataManagerImpl method listAllVolumes.

/**
 * @return list of all volumes.
 */
private List<OmVolumeArgs> listAllVolumes(String prefix, String startKey, int maxKeys) {
    List<OmVolumeArgs> result = Lists.newArrayList();
    /* volumeTable is full-cache, so we use cacheIterator. */
    Iterator<Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>>> cacheIterator = getVolumeTable().cacheIterator();
    String volumeName;
    OmVolumeArgs omVolumeArgs;
    boolean prefixIsEmpty = Strings.isNullOrEmpty(prefix);
    boolean startKeyIsEmpty = Strings.isNullOrEmpty(startKey);
    while (cacheIterator.hasNext() && result.size() < maxKeys) {
        Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> entry = cacheIterator.next();
        omVolumeArgs = entry.getValue().getCacheValue();
        volumeName = omVolumeArgs.getVolume();
        if (!prefixIsEmpty && !volumeName.startsWith(prefix)) {
            continue;
        }
        if (!startKeyIsEmpty) {
            if (volumeName.equals(startKey)) {
                startKeyIsEmpty = true;
            }
            continue;
        }
        result.add(omVolumeArgs);
    }
    return result;
}

Example 5 with CacheKey

use of org.apache.hadoop.hdds.utils.db.cache.CacheKey in project ozone by apache.

the class OmMetadataManagerImpl method isVolumeEmpty.

/**
 * Given a volume, check if it is empty, i.e there are no buckets inside it.
 * We iterate in the bucket table and see if there is any key that starts with
 * the volume prefix. We actually look for /volume/, since if we don't have
 * the trailing slash it is possible that we might match some other volume.
 * <p>
 * For example, vol1 and vol122 might match, to avoid that we look for /vol1/
 *
 * @param volume - Volume name
 * @return true if the volume is empty
 */
@Override
public boolean isVolumeEmpty(String volume) throws IOException {
    String volumePrefix = getVolumeKey(volume + OM_KEY_PREFIX);
    // First check in bucket table cache.
    Iterator<Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>>> iterator = ((TypedTable<String, OmBucketInfo>) bucketTable).cacheIterator();
    while (iterator.hasNext()) {
        Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> entry = iterator.next();
        String key = entry.getKey().getCacheKey();
        OmBucketInfo omBucketInfo = entry.getValue().getCacheValue();
        // Making sure that entry is not for delete bucket request.
        if (key.startsWith(volumePrefix) && omBucketInfo != null) {
            return false;
        }
    }
    try (TableIterator<String, ? extends KeyValue<String, OmBucketInfo>> bucketIter = bucketTable.iterator()) {
        KeyValue<String, OmBucketInfo> kv = bucketIter.seek(volumePrefix);
        if (kv != null) {
            // Check the entry in db is not marked for delete. This can happen
            // while entry is marked for delete, but it is not flushed to DB.
            CacheValue<OmBucketInfo> cacheValue = bucketTable.getCacheValue(new CacheKey(kv.getKey()));
            if (cacheValue != null) {
                if (kv.getKey().startsWith(volumePrefix) && cacheValue.getCacheValue() != null) {
                    // we found at least one bucket with this volume
                    return false;
                // prefix.
                }
            } else {
                if (kv.getKey().startsWith(volumePrefix)) {
                    // we found at least one bucket with this volume
                    return false;
                // prefix.
                }
            }
        }
    }
    return true;
}