use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.
the class ITListAzureBlobStorage method testListsAzureBlobStorageContent.
@Test
public void testListsAzureBlobStorageContent() throws InvalidKeyException, StorageException, URISyntaxException, IOException {
String containerName = String.format("%s-%s", AzureTestUtil.TEST_CONTAINER_NAME_PREFIX, UUID.randomUUID());
CloudBlobContainer container = AzureTestUtil.getContainer(containerName);
container.createIfNotExists();
CloudBlob blob = container.getBlockBlobReference(AzureTestUtil.TEST_BLOB_NAME);
byte[] buf = "0123456789".getBytes();
InputStream in = new ByteArrayInputStream(buf);
blob.upload(in, 10);
final TestRunner runner = TestRunners.newTestRunner(new ListAzureBlobStorage());
try {
runner.setProperty(AzureStorageUtils.ACCOUNT_NAME, AzureTestUtil.getAccountName());
runner.setProperty(AzureStorageUtils.ACCOUNT_KEY, AzureTestUtil.getAccountKey());
runner.setProperty(AzureStorageUtils.CONTAINER, containerName);
// requires multiple runs to deal with List processor checking
runner.run(3);
runner.assertTransferCount(ListAzureBlobStorage.REL_SUCCESS, 1);
runner.assertAllFlowFilesTransferred(ListAzureBlobStorage.REL_SUCCESS, 1);
for (MockFlowFile entry : runner.getFlowFilesForRelationship(ListAzureBlobStorage.REL_SUCCESS)) {
entry.assertAttributeEquals("azure.length", "10");
entry.assertAttributeEquals("mime.type", "application/octet-stream");
}
} finally {
container.deleteIfExists();
}
}
use of com.microsoft.azure.storage.blob.CloudBlob in project photon-model by vmware.
the class AzureComputeHostStorageStatsGatherer method getBlobUsedBytesAsync.
private void getBlobUsedBytesAsync(AzureStorageStatsDataHolder statsData, StorageMetricsStages next) {
Runnable getBlobsAsync = () -> {
String metricName = PhotonModelConstants.STORAGE_USED_BYTES;
List<ServiceStats.ServiceStat> statDatapoints = new ArrayList<>();
AtomicInteger accountsCount = new AtomicInteger(statsData.storageAccounts.size());
final List<Throwable> exs = new ArrayList<>();
for (Map.Entry<String, StorageAccount> account : statsData.storageAccounts.entrySet()) {
String resourceGroupName = getResourceGroupName(account.getValue().id);
statsData.azureClients.getAzureClient().storageAccounts().inner().listKeysAsync(resourceGroupName, account.getValue().name, new AzureAsyncCallback<StorageAccountListKeysResultInner>() {
@Override
public void onError(Throwable e) {
handleError(statsData, e);
}
@Override
public void onSuccess(StorageAccountListKeysResultInner result) {
logFine(() -> String.format("Retrieved the storage account keys for" + " storage account [%s].", account.getValue().name));
String storageConnectionString = String.format(STORAGE_CONNECTION_STRING, account.getValue().name, result.keys().get(0).value());
try {
CloudStorageAccount storageAccount = getAzureStorageClient(storageConnectionString);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
ResultContinuation nextContainerResults = null;
do {
ResultSegment<CloudBlobContainer> contSegment = blobClient.listContainersSegmented(null, ContainerListingDetails.NONE, QUERY_RESULT_LIMIT, nextContainerResults, null, null);
nextContainerResults = contSegment.getContinuationToken();
for (CloudBlobContainer container : contSegment.getResults()) {
ResultContinuation nextBlobResults = null;
do {
ResultSegment<ListBlobItem> blobsSegment = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), QUERY_RESULT_LIMIT, nextBlobResults, null, null);
nextBlobResults = blobsSegment.getContinuationToken();
for (ListBlobItem blobItem : blobsSegment.getResults()) {
if (blobItem instanceof CloudPageBlob) {
CloudPageBlob pageBlob = (CloudPageBlob) blobItem;
// TODO https://jira-hzn.eng.vmware.com/browse/VSYM-3445
try {
CloudBlob blobSnapshot = pageBlob.createSnapshot();
statsData.snapshots.add(blobSnapshot);
CloudPageBlob pageBlobSnapshot = (CloudPageBlob) blobSnapshot;
ArrayList<PageRange> pages = pageBlobSnapshot.downloadPageRanges();
// https://jira-hzn.eng.vmware.com/browse/VSYM-3355
for (PageRange pageRange : pages) {
statsData.utilizedBytes += pageRange.getEndOffset() - pageRange.getStartOffset();
}
} catch (StorageException e) {
logWarning(() -> String.format("Error getting blob size: [%s]", e.getMessage()));
}
}
}
} while (nextBlobResults != null);
}
} while (nextContainerResults != null);
} catch (Exception e) {
logWarning(() -> String.format("Exception while getting blob used bytes: %s", Utils.toString(e)));
exs.add(e);
} finally {
// in the Azure account
if (statsData.snapshots.size() > 0) {
synchronized (statsData.snapshots) {
Iterator<CloudBlob> snapshotIterator = statsData.snapshots.iterator();
while (snapshotIterator.hasNext()) {
try {
CloudBlob snapshot = snapshotIterator.next();
snapshot.deleteIfExists();
snapshotIterator.remove();
} catch (StorageException e) {
// Best effort to delete all the snapshots
logWarning(() -> String.format("Exception while deleting snapshot: %s", Utils.toString(e)));
}
}
}
}
}
// if all storage accounts were processed, create ServiceStat and finish
if (accountsCount.decrementAndGet() == 0) {
if (!exs.isEmpty()) {
handleError(statsData, exs.iterator().next());
return;
}
if (statsData.utilizedBytes != 0) {
ServiceStats.ServiceStat stat = new ServiceStats.ServiceStat();
stat.latestValue = statsData.utilizedBytes;
stat.sourceTimeMicrosUtc = TimeUnit.MILLISECONDS.toMicros(Utils.getNowMicrosUtc());
stat.unit = PhotonModelConstants.getUnitForMetric(metricName);
statDatapoints.add(stat);
}
statsData.statsResponse.statValues.put(metricName, statDatapoints);
if (statsData.statsResponse.statValues.size() == 1) {
statsData.statsResponse.computeLink = statsData.computeHostDesc.documentSelfLink;
}
statsData.stage = next;
handleStorageMetricDiscovery(statsData);
}
}
});
}
};
PhotonModelUtils.runInExecutor(this.executorService, getBlobsAsync, throwable -> handleError(statsData, throwable));
}
use of com.microsoft.azure.storage.blob.CloudBlob in project components by Talend.
the class AzureStorageGetRuntime method download.
private void download(RuntimeContainer runtimeContainer) {
FileOutputStream fos = null;
try {
List<RemoteBlobGet> remoteBlobs = createRemoteBlobsGet();
for (RemoteBlobGet rmtb : remoteBlobs) {
for (ListBlobItem blob : azureStorageBlobService.listBlobs(containerName, rmtb.prefix, rmtb.include)) {
if (blob instanceof CloudBlob) {
// TODO - Action when create is false and include is true ???
if (rmtb.create) {
new File(localFolder + "/" + ((CloudBlob) blob).getName()).getParentFile().mkdirs();
}
fos = new FileOutputStream(localFolder + "/" + ((CloudBlob) blob).getName());
azureStorageBlobService.download((CloudBlob) blob, fos);
}
}
}
} catch (StorageException | URISyntaxException | FileNotFoundException | InvalidKeyException e) {
LOGGER.error(e.getLocalizedMessage());
if (dieOnError) {
throw new ComponentException(e);
}
} finally {
try {
fos.close();
} catch (Exception e) {
// ignore
}
}
}
use of com.microsoft.azure.storage.blob.CloudBlob in project druid by druid-io.
the class AzureStorage method emptyCloudBlobDirectory.
public List<String> emptyCloudBlobDirectory(final String containerName, final String virtualDirPath) throws StorageException, URISyntaxException {
List<String> deletedFiles = new ArrayList<>();
CloudBlobContainer container = getOrCreateCloudBlobContainer(containerName);
for (ListBlobItem blobItem : container.listBlobs(virtualDirPath, true, null, null, null)) {
CloudBlob cloudBlob = (CloudBlob) blobItem;
log.info("Removing file[%s] from Azure.", cloudBlob.getName());
if (cloudBlob.deleteIfExists()) {
deletedFiles.add(cloudBlob.getName());
}
}
if (deletedFiles.isEmpty()) {
log.warn("No files were deleted on the following Azure path: [%s]", virtualDirPath);
}
return deletedFiles;
}
Aggregations