Examples with ListBlobItem com.microsoft.azure.storage.blob.ListBlobItem used on opensource projects

Example 1 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project camel by apache.

the class BlobServiceProducer method listBlobs.

private void listBlobs(Exchange exchange) throws Exception {
    CloudBlobContainer client = BlobServiceUtil.createBlobContainerClient(getConfiguration());
    BlobServiceRequestOptions opts = BlobServiceUtil.getRequestOptions(exchange);
    LOG.trace("Getting the blob list from the container [{}] from exchange [{}]...", getConfiguration().getContainerName(), exchange);
    BlobServiceConfiguration cfg = getConfiguration();
    EnumSet<BlobListingDetails> details = null;
    Object detailsObject = exchange.getIn().getHeader(BlobServiceConstants.BLOB_LISTING_DETAILS);
    if (detailsObject instanceof EnumSet) {
        @SuppressWarnings("unchecked") EnumSet<BlobListingDetails> theDetails = (EnumSet<BlobListingDetails>) detailsObject;
        details = theDetails;
    } else if (detailsObject instanceof BlobListingDetails) {
        details = EnumSet.of((BlobListingDetails) detailsObject);
    }
    Iterable<ListBlobItem> items = client.listBlobs(cfg.getBlobPrefix(), cfg.isUseFlatListing(), details, opts.getRequestOpts(), opts.getOpContext());
    ExchangeUtil.getMessageForResponse(exchange).setBody(items);
}

Example 2 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project hadoop by apache.

the class AzureNativeFileSystemStore method list.

private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException {
    try {
        checkContainer(ContainerAccessType.PureRead);
        if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        // Enable flat listing option only if depth is unbounded and config
        // KEY_ENABLE_FLAT_LISTING is enabled.
        boolean enableFlatListing = false;
        if (maxListingDepth < 0 && sessionConfiguration.getBoolean(KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
            enableFlatListing = true;
        }
        Iterable<ListBlobItem> objects;
        if (prefix.equals("/")) {
            objects = listRootBlobs(true, enableFlatListing);
        } else {
            objects = listRootBlobs(prefix, true, enableFlatListing);
        }
        ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
        for (ListBlobItem blobItem : objects) {
            //
            if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) {
                break;
            }
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the metadata to the list, but remove any existing duplicate
                // entries first that we may have added by finding nested files.
                FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
                if (existing != null) {
                    fileMetadata.remove(existing);
                }
                fileMetadata.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // Determine format of directory name depending on whether an absolute
                // path is being used or not.
                //
                String dirKey = normalizeKey(directory);
                // Strip the last /
                if (dirKey.endsWith(PATH_DELIMITER)) {
                    dirKey = dirKey.substring(0, dirKey.length() - 1);
                }
                // Reached the targeted listing depth. Return metadata for the
                // directory using default permissions.
                //
                // Note: Something smarter should be done about permissions. Maybe
                // inherit the permissions of the first non-directory blob.
                // Also, getting a proper value for last-modified is tricky.
                FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                // there.
                if (getFileMetadataInList(fileMetadata, dirKey) == null) {
                    fileMetadata.add(directoryMetadata);
                }
                if (!enableFlatListing) {
                    // Currently at a depth of one, decrement the listing depth for
                    // sub-directories.
                    buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1);
                }
            }
        }
        // Note: Original code indicated that this may be a hack.
        priorLastKey = null;
        PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix });
        return listing;
    } catch (Exception e) {
        //
        throw new AzureException(e);
    }
}

Example 3 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project hadoop by apache.

the class AzureNativeFileSystemStore method retrieveMetadata.

@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
    // server.
    if (null == storageInteractionLayer) {
        final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
        throw new AssertionError(errMsg);
    }
    LOG.debug("Retrieving metadata for {}", key);
    try {
        if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) {
            // return null now.
            return null;
        }
        // key is a container.
        if (key.equals("/")) {
            // Set the modification time for root to zero.
            return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
        }
        CloudBlobWrapper blob = getBlobReference(key);
        // exists.
        if (null != blob && blob.exists(getInstrumentedContext())) {
            LOG.debug("Found {} as an explicit blob. Checking if it's a file or folder.", key);
            // The blob exists, so capture the metadata from the blob
            // properties.
            blob.downloadAttributes(getInstrumentedContext());
            BlobProperties properties = blob.getProperties();
            if (retrieveFolderAttribute(blob)) {
                LOG.debug("{} is a folder blob.", key);
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
            } else {
                LOG.debug("{} is a normal blob.", key);
                return new FileMetadata(// Always return denormalized key with metadata.
                key, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
            }
        }
        // There is no file with that key name, but maybe it is a folder.
        // Query the underlying folder/container to list the blobs stored
        // there under that key.
        //
        Iterable<ListBlobItem> objects = listRootBlobs(key, true, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext());
        // Check if the directory/container has the blob items.
        for (ListBlobItem blobItem : objects) {
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                LOG.debug("Found blob as a directory-using this file under it to infer its properties {}", blobItem.getUri());
                blob = (CloudBlobWrapper) blobItem;
                // The key specifies a directory. Create a FileMetadata object which
                // specifies as such.
                BlobProperties properties = blob.getProperties();
                return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Implicit);
            }
        }
        // Return to caller with a null metadata object.
        return null;
    } catch (Exception e) {
        // Re-throw the exception as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 4 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project elasticsearch by elastic.

the class AzureStorageServiceImpl method deleteFiles.

@Override
public void deleteFiles(String account, LocationMode mode, String container, String path) throws URISyntaxException, StorageException {
    logger.trace("delete files container [{}], path [{}]", container, path);
    // Container name must be lower case.
    CloudBlobClient client = this.getSelectedClient(account, mode);
    CloudBlobContainer blobContainer = client.getContainerReference(container);
    SocketAccess.doPrivilegedVoidException(() -> {
        if (blobContainer.exists()) {
            // We list the blobs using a flat blob listing mode
            for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
                String blobName = blobNameFromUri(blobItem.getUri());
                logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
                deleteBlob(account, mode, container, blobName);
            }
        }
    });
}

Example 5 with ListBlobItem

use of com.microsoft.azure.storage.blob.ListBlobItem in project elasticsearch by elastic.

the class AzureStorageServiceImpl method listBlobsByPrefix.

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
    logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
    MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
    CloudBlobClient client = this.getSelectedClient(account, mode);
    CloudBlobContainer blobContainer = client.getContainerReference(container);
    SocketAccess.doPrivilegedVoidException(() -> {
        if (blobContainer.exists()) {
            for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) {
                URI uri = blobItem.getUri();
                logger.trace("blob url [{}]", uri);
                // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                String blobPath = uri.getPath().substring(1 + container.length() + 1);
                CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobPath);
                // fetch the blob attributes from Azure (getBlockBlobReference does not do this)
                // this is needed to retrieve the blob length (among other metadata) from Azure Storage
                blob.downloadAttributes();
                BlobProperties properties = blob.getProperties();
                String name = blobPath.substring(keyPath.length());
                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
            }
        }
    });
    return blobsBuilder.immutableMap();
}