Examples with CloudPageBlobWrapper org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper used on opensource projects

Example 1 with CloudPageBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method list.

private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException {
    try {
        checkContainer(ContainerAccessType.PureRead);
        if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        // Enable flat listing option only if depth is unbounded and config
        // KEY_ENABLE_FLAT_LISTING is enabled.
        boolean enableFlatListing = false;
        if (maxListingDepth < 0 && sessionConfiguration.getBoolean(KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
            enableFlatListing = true;
        }
        Iterable<ListBlobItem> objects;
        if (prefix.equals("/")) {
            objects = listRootBlobs(true, enableFlatListing);
        } else {
            objects = listRootBlobs(prefix, true, enableFlatListing);
        }
        ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
        for (ListBlobItem blobItem : objects) {
            //
            if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) {
                break;
            }
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the metadata to the list, but remove any existing duplicate
                // entries first that we may have added by finding nested files.
                FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
                if (existing != null) {
                    fileMetadata.remove(existing);
                }
                fileMetadata.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // Determine format of directory name depending on whether an absolute
                // path is being used or not.
                //
                String dirKey = normalizeKey(directory);
                // Strip the last /
                if (dirKey.endsWith(PATH_DELIMITER)) {
                    dirKey = dirKey.substring(0, dirKey.length() - 1);
                }
                // Reached the targeted listing depth. Return metadata for the
                // directory using default permissions.
                //
                // Note: Something smarter should be done about permissions. Maybe
                // inherit the permissions of the first non-directory blob.
                // Also, getting a proper value for last-modified is tricky.
                FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                // there.
                if (getFileMetadataInList(fileMetadata, dirKey) == null) {
                    fileMetadata.add(directoryMetadata);
                }
                if (!enableFlatListing) {
                    // Currently at a depth of one, decrement the listing depth for
                    // sub-directories.
                    buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1);
                }
            }
        }
        // Note: Original code indicated that this may be a hack.
        priorLastKey = null;
        PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix });
        return listing;
    } catch (Exception e) {
        //
        throw new AzureException(e);
    }
}

Example 2 with CloudPageBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method retrieveMetadata.

@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
    // server.
    if (null == storageInteractionLayer) {
        final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
        throw new AssertionError(errMsg);
    }
    LOG.debug("Retrieving metadata for {}", key);
    try {
        if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) {
            // return null now.
            return null;
        }
        // key is a container.
        if (key.equals("/")) {
            // Set the modification time for root to zero.
            return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
        }
        CloudBlobWrapper blob = getBlobReference(key);
        // exists.
        if (null != blob && blob.exists(getInstrumentedContext())) {
            LOG.debug("Found {} as an explicit blob. Checking if it's a file or folder.", key);
            // The blob exists, so capture the metadata from the blob
            // properties.
            blob.downloadAttributes(getInstrumentedContext());
            BlobProperties properties = blob.getProperties();
            if (retrieveFolderAttribute(blob)) {
                LOG.debug("{} is a folder blob.", key);
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
            } else {
                LOG.debug("{} is a normal blob.", key);
                return new FileMetadata(// Always return denormalized key with metadata.
                key, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
            }
        }
        // There is no file with that key name, but maybe it is a folder.
        // Query the underlying folder/container to list the blobs stored
        // there under that key.
        //
        Iterable<ListBlobItem> objects = listRootBlobs(key, true, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
        // Check if the directory/container has the blob items.
        for (ListBlobItem blobItem : objects) {
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                LOG.debug("Found blob as a directory-using this file under it to infer its properties {}", blobItem.getUri());
                blob = (CloudBlobWrapper) blobItem;
                // The key specifies a directory. Create a FileMetadata object which
                // specifies as such.
                BlobProperties properties = blob.getProperties();
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Implicit);
            }
        }
        // Return to caller with a null metadata object.
        return null;
    } catch (Exception e) {
        // Re-throw the exception as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 3 with CloudPageBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method buildUpList.

/**
   * Build up a metadata list of blobs in an Azure blob directory. This method
   * uses a in-order first traversal of blob directory structures to maintain
   * the sorted order of the blob names.
   * 
   * @param aCloudBlobDirectory Azure blob directory
   * @param aFileMetadataList a list of file metadata objects for each
   *                          non-directory blob.
   * @param maxListingCount maximum length of the built up list.
   */
private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory, ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount, final int maxListingDepth) throws Exception {
    // Push the blob directory onto the stack.
    //
    AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack = new AzureLinkedStack<Iterator<ListBlobItem>>();
    Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null, false, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
    Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
    if (0 == maxListingDepth || 0 == maxListingCount) {
        // immediately.
        return;
    }
    // The directory listing depth is unbounded if the maximum listing depth
    // is negative.
    final boolean isUnboundedDepth = (maxListingDepth < 0);
    // Reset the current directory listing depth.
    int listingDepth = 1;
    // metadata list is less than the max listing count.
    while (null != blobItemIterator && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) {
        while (blobItemIterator.hasNext()) {
            //
            if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) {
                break;
            }
            ListBlobItem blobItem = blobItemIterator.next();
            //
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the directory metadata to the list only if it's not already
                // there.
                FileMetadata existing = getFileMetadataInList(aFileMetadataList, blobKey);
                if (existing != null) {
                    aFileMetadataList.remove(existing);
                }
                aFileMetadataList.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // directory.
                if (isUnboundedDepth || maxListingDepth > listingDepth) {
                    // Push the current directory on the stack and increment the listing
                    // depth.
                    dirIteratorStack.push(blobItemIterator);
                    ++listingDepth;
                    // The current blob item represents the new directory. Get
                    // an iterator for this directory and continue by iterating through
                    // this directory.
                    blobItems = directory.listBlobs(null, false, EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext());
                    blobItemIterator = blobItems.iterator();
                } else {
                    // Determine format of directory name depending on whether an
                    // absolute path is being used or not.
                    String dirKey = normalizeKey(directory);
                    if (getFileMetadataInList(aFileMetadataList, dirKey) == null) {
                        // Reached the targeted listing depth. Return metadata for the
                        // directory using default permissions.
                        //
                        // Note: Something smarter should be done about permissions. Maybe
                        // inherit the permissions of the first non-directory blob.
                        // Also, getting a proper value for last-modified is tricky.
                        //
                        FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                        // Add the directory metadata to the list.
                        aFileMetadataList.add(directoryMetadata);
                    }
                }
            }
        }
        //
        if (dirIteratorStack.isEmpty()) {
            blobItemIterator = null;
        } else {
            // Pop the next directory item from the stack and decrement the
            // depth.
            blobItemIterator = dirIteratorStack.pop();
            --listingDepth;
            // Assertion: Listing depth should not be less than zero.
            if (listingDepth < 0) {
                throw new AssertionError("Non-negative listing depth expected");
            }
        }
    }
}