Examples with CloudBlobDirectoryWrapper org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper used on opensource projects

Example 1 with CloudBlobDirectoryWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method list.

private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException {
    try {
        checkContainer(ContainerAccessType.PureRead);
        if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        // Enable flat listing option only if depth is unbounded and config
        // KEY_ENABLE_FLAT_LISTING is enabled.
        boolean enableFlatListing = false;
        if (maxListingDepth < 0 && sessionConfiguration.getBoolean(KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
            enableFlatListing = true;
        }
        Iterable<ListBlobItem> objects;
        if (prefix.equals("/")) {
            objects = listRootBlobs(true, enableFlatListing);
        } else {
            objects = listRootBlobs(prefix, true, enableFlatListing);
        }
        ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
        for (ListBlobItem blobItem : objects) {
            //
            if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) {
                break;
            }
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the metadata to the list, but remove any existing duplicate
                // entries first that we may have added by finding nested files.
                FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
                if (existing != null) {
                    fileMetadata.remove(existing);
                }
                fileMetadata.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // Determine format of directory name depending on whether an absolute
                // path is being used or not.
                //
                String dirKey = normalizeKey(directory);
                // Strip the last /
                if (dirKey.endsWith(PATH_DELIMITER)) {
                    dirKey = dirKey.substring(0, dirKey.length() - 1);
                }
                // Reached the targeted listing depth. Return metadata for the
                // directory using default permissions.
                //
                // Note: Something smarter should be done about permissions. Maybe
                // inherit the permissions of the first non-directory blob.
                // Also, getting a proper value for last-modified is tricky.
                FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                // there.
                if (getFileMetadataInList(fileMetadata, dirKey) == null) {
                    fileMetadata.add(directoryMetadata);
                }
                if (!enableFlatListing) {
                    // Currently at a depth of one, decrement the listing depth for
                    // sub-directories.
                    buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1);
                }
            }
        }
        // Note: Original code indicated that this may be a hack.
        priorLastKey = null;
        PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix });
        return listing;
    } catch (Exception e) {
        //
        throw new AzureException(e);
    }
}

Example 2 with CloudBlobDirectoryWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method buildUpList.

/**
   * Build up a metadata list of blobs in an Azure blob directory. This method
   * uses a in-order first traversal of blob directory structures to maintain
   * the sorted order of the blob names.
   * 
   * @param aCloudBlobDirectory Azure blob directory
   * @param aFileMetadataList a list of file metadata objects for each
   *                          non-directory blob.
   * @param maxListingCount maximum length of the built up list.
   */
private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory, ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount, final int maxListingDepth) throws Exception {
    // Push the blob directory onto the stack.
    //
    AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack = new AzureLinkedStack<Iterator<ListBlobItem>>();
    Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null, false, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
    Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
    if (0 == maxListingDepth || 0 == maxListingCount) {
        // immediately.
        return;
    }
    // The directory listing depth is unbounded if the maximum listing depth
    // is negative.
    final boolean isUnboundedDepth = (maxListingDepth < 0);
    // Reset the current directory listing depth.
    int listingDepth = 1;
    // metadata list is less than the max listing count.
    while (null != blobItemIterator && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) {
        while (blobItemIterator.hasNext()) {
            //
            if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) {
                break;
            }
            ListBlobItem blobItem = blobItemIterator.next();
            //
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the directory metadata to the list only if it's not already
                // there.
                FileMetadata existing = getFileMetadataInList(aFileMetadataList, blobKey);
                if (existing != null) {
                    aFileMetadataList.remove(existing);
                }
                aFileMetadataList.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // directory.
                if (isUnboundedDepth || maxListingDepth > listingDepth) {
                    // Push the current directory on the stack and increment the listing
                    // depth.
                    dirIteratorStack.push(blobItemIterator);
                    ++listingDepth;
                    // The current blob item represents the new directory. Get
                    // an iterator for this directory and continue by iterating through
                    // this directory.
                    blobItems = directory.listBlobs(null, false, EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext());
                    blobItemIterator = blobItems.iterator();
                } else {
                    // Determine format of directory name depending on whether an
                    // absolute path is being used or not.
                    String dirKey = normalizeKey(directory);
                    if (getFileMetadataInList(aFileMetadataList, dirKey) == null) {
                        // Reached the targeted listing depth. Return metadata for the
                        // directory using default permissions.
                        //
                        // Note: Something smarter should be done about permissions. Maybe
                        // inherit the permissions of the first non-directory blob.
                        // Also, getting a proper value for last-modified is tricky.
                        //
                        FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                        // Add the directory metadata to the list.
                        aFileMetadataList.add(directoryMetadata);
                    }
                }
            }
        }
        //
        if (dirIteratorStack.isEmpty()) {
            blobItemIterator = null;
        } else {
            // Pop the next directory item from the stack and decrement the
            // depth.
            blobItemIterator = dirIteratorStack.pop();
            --listingDepth;
            // Assertion: Listing depth should not be less than zero.
            if (listingDepth < 0) {
                throw new AssertionError("Non-negative listing depth expected");
            }
        }
    }
}