Examples with BlobProperties com.microsoft.azure.storage.blob.BlobProperties used on opensource projects

Example 1 with BlobProperties

use of com.microsoft.azure.storage.blob.BlobProperties in project hadoop by apache.

the class AzureNativeFileSystemStore method list.

private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException {
    try {
        checkContainer(ContainerAccessType.PureRead);
        if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        // Enable flat listing option only if depth is unbounded and config
        // KEY_ENABLE_FLAT_LISTING is enabled.
        boolean enableFlatListing = false;
        if (maxListingDepth < 0 && sessionConfiguration.getBoolean(KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
            enableFlatListing = true;
        }
        Iterable<ListBlobItem> objects;
        if (prefix.equals("/")) {
            objects = listRootBlobs(true, enableFlatListing);
        } else {
            objects = listRootBlobs(prefix, true, enableFlatListing);
        }
        ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
        for (ListBlobItem blobItem : objects) {
            //
            if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) {
                break;
            }
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the metadata to the list, but remove any existing duplicate
                // entries first that we may have added by finding nested files.
                FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
                if (existing != null) {
                    fileMetadata.remove(existing);
                }
                fileMetadata.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // Determine format of directory name depending on whether an absolute
                // path is being used or not.
                //
                String dirKey = normalizeKey(directory);
                // Strip the last /
                if (dirKey.endsWith(PATH_DELIMITER)) {
                    dirKey = dirKey.substring(0, dirKey.length() - 1);
                }
                // Reached the targeted listing depth. Return metadata for the
                // directory using default permissions.
                //
                // Note: Something smarter should be done about permissions. Maybe
                // inherit the permissions of the first non-directory blob.
                // Also, getting a proper value for last-modified is tricky.
                FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                // there.
                if (getFileMetadataInList(fileMetadata, dirKey) == null) {
                    fileMetadata.add(directoryMetadata);
                }
                if (!enableFlatListing) {
                    // Currently at a depth of one, decrement the listing depth for
                    // sub-directories.
                    buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1);
                }
            }
        }
        // Note: Original code indicated that this may be a hack.
        priorLastKey = null;
        PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix });
        return listing;
    } catch (Exception e) {
        //
        throw new AzureException(e);
    }
}

Example 2 with BlobProperties

use of com.microsoft.azure.storage.blob.BlobProperties in project hadoop by apache.

the class AzureNativeFileSystemStore method retrieveMetadata.

@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
    // server.
    if (null == storageInteractionLayer) {
        final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
        throw new AssertionError(errMsg);
    }
    LOG.debug("Retrieving metadata for {}", key);
    try {
        if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) {
            // return null now.
            return null;
        }
        // key is a container.
        if (key.equals("/")) {
            // Set the modification time for root to zero.
            return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
        }
        CloudBlobWrapper blob = getBlobReference(key);
        // exists.
        if (null != blob && blob.exists(getInstrumentedContext())) {
            LOG.debug("Found {} as an explicit blob. Checking if it's a file or folder.", key);
            // The blob exists, so capture the metadata from the blob
            // properties.
            blob.downloadAttributes(getInstrumentedContext());
            BlobProperties properties = blob.getProperties();
            if (retrieveFolderAttribute(blob)) {
                LOG.debug("{} is a folder blob.", key);
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
            } else {
                LOG.debug("{} is a normal blob.", key);
                return new FileMetadata(// Always return denormalized key with metadata.
                key, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
            }
        }
        // There is no file with that key name, but maybe it is a folder.
        // Query the underlying folder/container to list the blobs stored
        // there under that key.
        //
        Iterable<ListBlobItem> objects = listRootBlobs(key, true, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
        // Check if the directory/container has the blob items.
        for (ListBlobItem blobItem : objects) {
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                LOG.debug("Found blob as a directory-using this file under it to infer its properties {}", blobItem.getUri());
                blob = (CloudBlobWrapper) blobItem;
                // The key specifies a directory. Create a FileMetadata object which
                // specifies as such.
                BlobProperties properties = blob.getProperties();
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Implicit);
            }
        }
        // Return to caller with a null metadata object.
        return null;
    } catch (Exception e) {
        // Re-throw the exception as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 3 with BlobProperties

use of com.microsoft.azure.storage.blob.BlobProperties in project elasticsearch by elastic.

the class AzureStorageServiceImpl method listBlobsByPrefix.

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
    logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
    MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
    CloudBlobClient client = this.getSelectedClient(account, mode);
    CloudBlobContainer blobContainer = client.getContainerReference(container);
    SocketAccess.doPrivilegedVoidException(() -> {
        if (blobContainer.exists()) {
            for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) {
                URI uri = blobItem.getUri();
                logger.trace("blob url [{}]", uri);
                // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                String blobPath = uri.getPath().substring(1 + container.length() + 1);
                CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobPath);
                // fetch the blob attributes from Azure (getBlockBlobReference does not do this)
                // this is needed to retrieve the blob length (among other metadata) from Azure Storage
                blob.downloadAttributes();
                BlobProperties properties = blob.getProperties();
                String name = blobPath.substring(keyPath.length());
                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
            }
        }
    });
    return blobsBuilder.immutableMap();
}

Example 4 with BlobProperties

use of com.microsoft.azure.storage.blob.BlobProperties in project hadoop by apache.

the class AzureNativeFileSystemStore method buildUpList.

/**
   * Build up a metadata list of blobs in an Azure blob directory. This method
   * uses a in-order first traversal of blob directory structures to maintain
   * the sorted order of the blob names.
   * 
   * @param aCloudBlobDirectory Azure blob directory
   * @param aFileMetadataList a list of file metadata objects for each
   *                          non-directory blob.
   * @param maxListingCount maximum length of the built up list.
   */
private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory, ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount, final int maxListingDepth) throws Exception {
    // Push the blob directory onto the stack.
    //
    AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack = new AzureLinkedStack<Iterator<ListBlobItem>>();
    Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null, false, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
    Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
    if (0 == maxListingDepth || 0 == maxListingCount) {
        // immediately.
        return;
    }
    // The directory listing depth is unbounded if the maximum listing depth
    // is negative.
    final boolean isUnboundedDepth = (maxListingDepth < 0);
    // Reset the current directory listing depth.
    int listingDepth = 1;
    // metadata list is less than the max listing count.
    while (null != blobItemIterator && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) {
        while (blobItemIterator.hasNext()) {
            //
            if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) {
                break;
            }
            ListBlobItem blobItem = blobItemIterator.next();
            //
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the directory metadata to the list only if it's not already
                // there.
                FileMetadata existing = getFileMetadataInList(aFileMetadataList, blobKey);
                if (existing != null) {
                    aFileMetadataList.remove(existing);
                }
                aFileMetadataList.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // directory.
                if (isUnboundedDepth || maxListingDepth > listingDepth) {
                    // Push the current directory on the stack and increment the listing
                    // depth.
                    dirIteratorStack.push(blobItemIterator);
                    ++listingDepth;
                    // The current blob item represents the new directory. Get
                    // an iterator for this directory and continue by iterating through
                    // this directory.
                    blobItems = directory.listBlobs(null, false, EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext());
                    blobItemIterator = blobItems.iterator();
                } else {
                    // Determine format of directory name depending on whether an
                    // absolute path is being used or not.
                    String dirKey = normalizeKey(directory);
                    if (getFileMetadataInList(aFileMetadataList, dirKey) == null) {
                        // Reached the targeted listing depth. Return metadata for the
                        // directory using default permissions.
                        //
                        // Note: Something smarter should be done about permissions. Maybe
                        // inherit the permissions of the first non-directory blob.
                        // Also, getting a proper value for last-modified is tricky.
                        //
                        FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                        // Add the directory metadata to the list.
                        aFileMetadataList.add(directoryMetadata);
                    }
                }
            }
        }
        //
        if (dirIteratorStack.isEmpty()) {
            blobItemIterator = null;
        } else {
            // Pop the next directory item from the stack and decrement the
            // depth.
            blobItemIterator = dirIteratorStack.pop();
            --listingDepth;
            // Assertion: Listing depth should not be less than zero.
            if (listingDepth < 0) {
                throw new AssertionError("Non-negative listing depth expected");
            }
        }
    }
}