use of com.azure.storage.blob.models.BlobListDetails in project kafka-connect-file-pulse by streamthoughts.
the class AzureBlobStorageFileSystemListing method listObjects.
/**
* {@inheritDoc}
*/
@Override
public Collection<FileObjectMeta> listObjects() {
LOG.debug("Listing objects in container '{}' using prefix '{}'", config.getContainerName(), config.getPrefix());
final BlobContainerClient blobContainerClient = storage.getBlobContainerClient();
final ListBlobsOptions options = new ListBlobsOptions().setDetails(new BlobListDetails().setRetrieveMetadata(true));
if (isNotBlank(config.getPrefix())) {
options.setPrefix(config.getPrefix());
}
final PagedIterable<BlobItem> blobItems = blobContainerClient.listBlobs(options, DEFAULT_TIME);
final List<FileObjectMeta> fileObjectMetaList = new LinkedList<>();
for (final BlobItem item : blobItems) {
LOG.debug("Find BlobItem with name '{}'", item.getName());
final Boolean prefix = item.isPrefix();
if (prefix != null && prefix) {
LOG.info("Ignored virtual directory prefix: '{}'", item.getName());
} else {
final BlobClient blobClient = blobContainerClient.getBlobClient(item.getName());
if (isDirectory(blobClient)) {
LOG.info("Ignored virtual directory prefix: '{}'", item.getName());
} else {
final GenericFileObjectMeta objectMetadata = storage.getObjectMetadata(blobClient);
fileObjectMetaList.add(objectMetadata);
}
}
}
return filter == null ? fileObjectMetaList : filter.filterFiles(fileObjectMetaList);
}
use of com.azure.storage.blob.models.BlobListDetails in project OpenSearch by opensearch-project.
the class AzureBlobStore method children.
public Map<String, BlobContainer> children(BlobPath path) throws URISyntaxException, BlobStorageException {
final Set<String> blobsBuilder = new HashSet<String>();
final Tuple<BlobServiceClient, Supplier<Context>> client = client();
final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
final String keyPath = path.buildAsString();
final ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(new BlobListDetails().setRetrieveMetadata(true)).setPrefix(keyPath);
SocketAccess.doPrivilegedVoidException(() -> {
for (final BlobItem blobItem : blobContainer.listBlobsByHierarchy("/", listBlobsOptions, timeout())) {
// Skipping over the blobs, only look for prefixes
if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
// Expecting name in the form /container/keyPath.* and we want to strip off the /container/
// this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
// Lastly, we add the length of keyPath to the offset to strip this container's path.
final String name = getBlobName(blobItem.getName(), container, keyPath).replaceAll("/$", "");
logger.trace(() -> new ParameterizedMessage("blob name [{}]", name));
blobsBuilder.add(name);
}
}
});
return Collections.unmodifiableMap(blobsBuilder.stream().collect(Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this, threadPool))));
}
use of com.azure.storage.blob.models.BlobListDetails in project OpenSearch by opensearch-project.
the class AzureBlobStore method listBlobsByPrefix.
public Map<String, BlobMetadata> listBlobsByPrefix(String keyPath, String prefix) throws URISyntaxException, BlobStorageException {
final Map<String, BlobMetadata> blobsBuilder = new HashMap<String, BlobMetadata>();
final Tuple<BlobServiceClient, Supplier<Context>> client = client();
final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
logger.trace(() -> new ParameterizedMessage("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix));
// NOTE: this should be here: if (prefix == null) prefix = "";
// however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
// then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
final ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(new BlobListDetails().setRetrieveMetadata(true)).setPrefix(keyPath + (prefix == null ? "" : prefix));
SocketAccess.doPrivilegedVoidException(() -> {
for (final BlobItem blobItem : blobContainer.listBlobsByHierarchy("/", listBlobsOptions, timeout())) {
// Skipping over the prefixes, only look for the blobs
if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
continue;
}
final String name = getBlobName(blobItem.getName(), container, keyPath);
logger.trace(() -> new ParameterizedMessage("blob name [{}]", name));
final BlobItemProperties properties = blobItem.getProperties();
logger.trace(() -> new ParameterizedMessage("blob name [{}], size [{}]", name, properties.getContentLength()));
blobsBuilder.put(name, new PlainBlobMetadata(name, properties.getContentLength()));
}
});
return MapBuilder.newMapBuilder(blobsBuilder).immutableMap();
}
use of com.azure.storage.blob.models.BlobListDetails in project openvsx by eclipse.
the class AzureDownloadCountService method listBlobs.
private PagedIterable<BlobItem> listBlobs() {
var details = new BlobListDetails().setRetrieveCopy(false).setRetrieveMetadata(false).setRetrieveDeletedBlobs(false).setRetrieveTags(false).setRetrieveSnapshots(false).setRetrieveUncommittedBlobs(false).setRetrieveVersions(false);
var options = new ListBlobsOptions().setMaxResultsPerPage(100).setDetails(details);
return getContainerClient().listBlobs(options, Duration.ofMinutes(5));
}
Aggregations