Examples with ListBlobItem com.microsoft.azure.storage.blob.ListBlobItem used on opensource projects

Example 11 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project hadoop by apache.

the class AzureNativeFileSystemStore method buildUpList.

/**
   * Build up a metadata list of blobs in an Azure blob directory. This method
   * uses a in-order first traversal of blob directory structures to maintain
   * the sorted order of the blob names.
   * 
   * @param aCloudBlobDirectory Azure blob directory
   * @param aFileMetadataList a list of file metadata objects for each
   *                          non-directory blob.
   * @param maxListingCount maximum length of the built up list.
   */
private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory, ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount, final int maxListingDepth) throws Exception {
    // Push the blob directory onto the stack.
    //
    AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack = new AzureLinkedStack<Iterator<ListBlobItem>>();
    Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null, false, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
    Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
    if (0 == maxListingDepth || 0 == maxListingCount) {
        // immediately.
        return;
    }
    // The directory listing depth is unbounded if the maximum listing depth
    // is negative.
    final boolean isUnboundedDepth = (maxListingDepth < 0);
    // Reset the current directory listing depth.
    int listingDepth = 1;
    // metadata list is less than the max listing count.
    while (null != blobItemIterator && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) {
        while (blobItemIterator.hasNext()) {
            //
            if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) {
                break;
            }
            ListBlobItem blobItem = blobItemIterator.next();
            //
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the directory metadata to the list only if it's not already
                // there.
                FileMetadata existing = getFileMetadataInList(aFileMetadataList, blobKey);
                if (existing != null) {
                    aFileMetadataList.remove(existing);
                }
                aFileMetadataList.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // directory.
                if (isUnboundedDepth || maxListingDepth > listingDepth) {
                    // Push the current directory on the stack and increment the listing
                    // depth.
                    dirIteratorStack.push(blobItemIterator);
                    ++listingDepth;
                    // The current blob item represents the new directory. Get
                    // an iterator for this directory and continue by iterating through
                    // this directory.
                    blobItems = directory.listBlobs(null, false, EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext());
                    blobItemIterator = blobItems.iterator();
                } else {
                    // Determine format of directory name depending on whether an
                    // absolute path is being used or not.
                    String dirKey = normalizeKey(directory);
                    if (getFileMetadataInList(aFileMetadataList, dirKey) == null) {
                        // Reached the targeted listing depth. Return metadata for the
                        // directory using default permissions.
                        //
                        // Note: Something smarter should be done about permissions. Maybe
                        // inherit the permissions of the first non-directory blob.
                        // Also, getting a proper value for last-modified is tricky.
                        //
                        FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                        // Add the directory metadata to the list.
                        aFileMetadataList.add(directoryMetadata);
                    }
                }
            }
        }
        //
        if (dirIteratorStack.isEmpty()) {
            blobItemIterator = null;
        } else {
            // Pop the next directory item from the stack and decrement the
            // depth.
            blobItemIterator = dirIteratorStack.pop();
            --listingDepth;
            // Assertion: Listing depth should not be less than zero.
            if (listingDepth < 0) {
                throw new AssertionError("Non-negative listing depth expected");
            }
        }
    }
}