Examples with BlobItem com.azure.storage.blob.models.BlobItem used on opensource projects

Example 1 with BlobItem

use of com.azure.storage.blob.models.BlobItem in project beam by apache.

the class AzureBlobStoreFileSystem method expand.

/**
 * Expands a pattern into {@link MatchResult}.
 */
@VisibleForTesting
MatchResult expand(AzfsResourceId azfsPattern) {
    checkArgument(azfsPattern.isWildcard(), "The resource id should be a wildcard.");
    String blobPrefix = azfsPattern.getBlobNonWildcardPrefix();
    Pattern wildcardAsRegexp = Pattern.compile(wildcardToRegexp(azfsPattern.getBlob()));
    LOG.debug("matching files in container {}, prefix {} against pattern {}", azfsPattern.getContainer(), blobPrefix, wildcardAsRegexp.toString());
    ListBlobsOptions listOptions = new ListBlobsOptions().setPrefix(blobPrefix);
    Duration timeout = Duration.ofMinutes(1);
    String account = azfsPattern.getAccount();
    String container = azfsPattern.getContainer();
    BlobContainerClient blobContainerClient = client.get().getBlobContainerClient(container);
    PagedIterable<BlobItem> blobs = blobContainerClient.listBlobs(listOptions, timeout);
    List<MatchResult.Metadata> results = new ArrayList<>();
    blobs.forEach(blob -> {
        String name = blob.getName();
        if (wildcardAsRegexp.matcher(name).matches() && !name.endsWith("/")) {
            LOG.debug("Matched object: azfs://{}/{}/{}", account, container, name);
            BlobProperties properties = blobContainerClient.getBlobClient(name).getProperties();
            AzfsResourceId rid = AzfsResourceId.fromComponents(account, container, name).withSize(properties.getBlobSize()).withLastModified(Date.from(properties.getLastModified().toInstant()));
            results.add(toMetadata(rid, properties.getContentEncoding(), properties.getETag()));
        }
    });
    return MatchResult.create(MatchResult.Status.OK, results);
}

Example 2 with BlobItem

use of com.azure.storage.blob.models.BlobItem in project azure-iot-sdk-java by Azure.

the class ExportImportTests method runExportJob.

private static List<ExportImportDevice> runExportJob(Optional<StorageAuthenticationType> storageAuthenticationType) throws Exception {
    Boolean excludeKeys = false;
    String containerSasUri = getContainerSasUri(exportContainer);
    boolean exportJobScheduled = false;
    JobProperties exportJob = null;
    while (!exportJobScheduled) {
        try {
            if (storageAuthenticationType.isPresent()) {
                JobProperties exportJobProperties = JobProperties.createForExportJob(containerSasUri, excludeKeys, storageAuthenticationType.get());
                exportJob = registryManager.exportDevices(exportJobProperties);
            } else {
                exportJob = registryManager.exportDevices(containerSasUri, excludeKeys);
            }
            exportJobScheduled = true;
        } catch (IotHubTooManyDevicesException e) {
            // test is being throttled, wait a while and try again
            Thread.sleep(10 * 1000);
        }
    }
    JobProperties.JobStatus jobStatus;
    long startTime = System.currentTimeMillis();
    while (true) {
        exportJob = registryManager.getJob(exportJob.getJobId());
        jobStatus = exportJob.getStatus();
        if (jobStatus == JobProperties.JobStatus.COMPLETED || jobStatus == JobProperties.JobStatus.FAILED) {
            break;
        }
        if (System.currentTimeMillis() - startTime > EXPORT_JOB_TIMEOUT_MILLISECONDS) {
            fail("Timed out waiting for the export job to complete");
        }
        Thread.sleep(100);
    }
    String exportedDevicesJson = "";
    for (BlobItem blobItem : exportContainer.listBlobs()) {
        BlobInputStream stream = exportContainer.getBlobClient(blobItem.getName()).openInputStream();
        try (Scanner scanner = new Scanner(stream, StandardCharsets.UTF_8.name())) {
            exportedDevicesJson = scanner.next();
        }
    }
    List<ExportImportDevice> result = new ArrayList<>();
    Scanner scanner = new Scanner(exportedDevicesJson);
    while (scanner.hasNextLine()) {
        String exportImportDeviceJson = scanner.nextLine();
        ExportImportDeviceParser parser = new ExportImportDeviceParser(exportImportDeviceJson);
        ExportImportDevice device = Deencapsulation.newInstance(ExportImportDevice.class, new Class[] { ExportImportDeviceParser.class }, parser);
        device.setImportMode(ImportMode.CreateOrUpdate);
        result.add(device);
    }
    scanner.close();
    if (jobStatus != JobProperties.JobStatus.COMPLETED) {
        Assert.fail("The export job was not completed successfully");
    }
    return result;
}

Example 3 with BlobItem

use of com.azure.storage.blob.models.BlobItem in project beam by apache.

the class AzureBlobStoreFileSystem method delete.

/**
 * This method will delete a virtual folder or a blob, not a container.
 */
@Override
protected void delete(Collection<AzfsResourceId> resourceIds) throws IOException {
    for (AzfsResourceId resourceId : resourceIds) {
        if (resourceId.getBlob() == null) {
            throw new IOException("delete does not delete containers.");
        }
        BlobContainerClient container = client.get().getBlobContainerClient(resourceId.getContainer());
        // deleting a blob that is not a directory
        if (!resourceId.isDirectory()) {
            BlobClient blob = container.getBlobClient(resourceId.getBlob());
            if (!blob.exists()) {
                throw new FileNotFoundException("The resource to delete does not exist.");
            }
            blob.delete();
        } else // deleting a directory (not a container)
        {
            PagedIterable<BlobItem> blobsInDirectory = container.listBlobsByHierarchy(resourceId.getBlob());
            blobsInDirectory.forEach(blob -> {
                String blobName = blob.getName();
                container.getBlobClient(blobName).delete();
            });
        }
    }
}

Example 4 with BlobItem

use of com.azure.storage.blob.models.BlobItem in project ats-framework by Axway.

the class BlobStorageOperations method listBlobs.

/**
 * List blobs from container
 * @param containerName - the container name
 * @param prefix - prefix for the blobs names or null for all blobs
 * @param directory - the directory which will be listed, or null to search the whole container
 * @param retrieveTimeout - the maximum amount of time (in seconds) to wait for the operation to complete. If the operation did not complete in that time {@link BlobStorageException} will be thrown/raised. Pass 0 (zero) to use the default value
 * @return list {@link BlobInfo}
 * @throws AtsBlobStorageException - if exception occurred
 */
@PublicAtsApi
public List<BlobInfo> listBlobs(String containerName, String prefix, String directory, long retrieveTimeout) {
    try {
        StringBuilder sb = new StringBuilder();
        sb.append("Listing blobs");
        log.info(sb.toString());
        final List<BlobInfo> infos = new ArrayList<BlobInfo>();
        PagedIterable<BlobItem> blobs = null;
        ListBlobsOptions lbops = new ListBlobsOptions();
        if (!StringUtils.isNullOrEmpty(prefix)) {
            lbops.setPrefix(prefix);
            sb.append(" with prefix '" + prefix + "'");
        }
        if (retrieveTimeout <= 0) {
            // just a little less than too much
            retrieveTimeout = Integer.MAX_VALUE / 2;
        }
        if (!StringUtils.isNullOrEmpty(directory)) {
            String newPrefix = directory + "/" + prefix;
            lbops.setPrefix(newPrefix);
            sb.append(" from directory '" + directory + "'");
            sb.append(" in container '" + containerName + "' ...");
            log.info(sb.toString());
            blobs = serviceClient.getBlobContainerClient(containerName).listBlobsByHierarchy("/", lbops, Duration.ofSeconds(retrieveTimeout));
        } else {
            sb.append(" in container '" + containerName + "' ...");
            log.info(sb.toString());
            blobs = serviceClient.getBlobContainerClient(containerName).listBlobs(lbops, Duration.ofSeconds(retrieveTimeout));
        }
        if (blobs != null) {
            blobs.stream().forEach(new Consumer<BlobItem>() {

                public void accept(BlobItem blobItem) {
                    BlobInfo info = new BlobInfo();
                    BlobItemProperties properties = blobItem.getProperties();
                    info.setAccessTier(BlobInfo.toAtsAccessTier(properties.getAccessTier()));
                    info.setBlobName(blobItem.getName());
                    info.setBlobType(BlobInfo.toAtsBlobType(properties.getBlobType()));
                    info.setContainerName(containerName);
                    info.setContentType(properties.getContentType());
                    if (properties.getCreationTime() != null) {
                        info.setCreationTime(Date.from(properties.getCreationTime().toInstant()));
                    } else {
                        info.setCreationTime(null);
                    }
                    info.setETag(properties.getETag());
                    if (properties.getLastModified() != null) {
                        info.setLastModified(Date.from(properties.getLastModified().toInstant()));
                    } else {
                        info.setLastModified(null);
                    }
                    if (properties.getContentMd5() != null) {
                        info.setMd5(java.util.Base64.getEncoder().encodeToString(properties.getContentMd5()));
                    } else {
                        info.setMd5(null);
                    }
                    info.setMetadata(blobItem.getMetadata());
                    info.setSize(properties.getContentLength());
                    infos.add(info);
                }
            });
        }
        return infos;
    } catch (Exception e) {
        String errorMessage = "Could not list blobs from container '" + containerName + "'" + (!StringUtils.isNullOrEmpty(prefix) ? " with prefix '" + prefix + "'" : "") + (!StringUtils.isNullOrEmpty(directory) ? " in directory '" + directory + "'" : "") + " in " + retrieveTimeout + " seconds";
        throw new AtsBlobStorageException(errorMessage, e);
    }
}