use of com.microsoft.azure.management.Azure in project photon-model by vmware.
the class AzureStorageEnumerationAdapterService method createStorageDescriptions.
/*
* Create all storage descriptions Storage descriptions mapping to Azure storage accounts have
* an additional custom property {"storageType" : "Microsoft.Storage/storageAccounts"}
*/
private void createStorageDescriptions(StorageEnumContext context, StorageEnumStages next) {
if (context.storageAccountsToUpdateCreate.size() == 0) {
if (context.enumNextPageLink != null) {
context.subStage = StorageEnumStages.GET_STORAGE_ACCOUNTS;
handleSubStage(context);
return;
}
logFine(() -> "No storage account found for creation");
context.subStage = next;
handleSubStage(context);
return;
}
logFine(() -> String.format("%d storage description to be created", context.storageAccountsToUpdateCreate.size()));
Consumer<Throwable> failure = e -> {
logWarning("Failure getting Azure storage accounts [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
handleError(context, e);
};
PhotonModelUtils.runInExecutor(this.executorService, () -> {
StorageAccountsInner stOps = context.azureSdkClients.getAzureClient().storageAccounts().inner();
List<DeferredResult<StorageDescription>> results = context.storageAccountsToUpdateCreate.values().stream().map(sa -> createStorageDescription(context, sa, stOps)).collect(java.util.stream.Collectors.toList());
DeferredResult.allOf(results).whenComplete((sds, e) -> {
if (e != null) {
logWarning(() -> String.format("Error: %s", e.getMessage()));
}
context.subStage = next;
handleSubStage(context);
});
}, failure);
}
use of com.microsoft.azure.management.Azure in project photon-model by vmware.
the class AzureStorageEnumerationAdapterService method getStorageContainersAsync.
/*
* Get all Azure containers by storage account
*/
public void getStorageContainersAsync(StorageEnumContext context, StorageEnumStages next) {
if (context.storageAccountIds.size() == 0) {
logFine(() -> "No storage description available - clean up all resources");
context.subStage = StorageEnumStages.DISASSOCIATE_RESOURCE_GROUP_STATES;
handleSubStage(context);
return;
}
Consumer<Throwable> failure = e -> {
logWarning("Failure getting Azure storage containers [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
handleError(context, e);
};
PhotonModelUtils.runInExecutor(this.executorService, () -> {
for (String id : context.storageAccountIds) {
String storageConnectionString = context.storageConnectionStrings.get(id);
if (storageConnectionString == null) {
continue;
}
try {
CloudStorageAccount storageAccount;
if (AzureUtils.isAzureClientMock()) {
StorageCredentials credentials = StorageCredentials.tryParseCredentials(storageConnectionString);
storageAccount = new CloudStorageAccount(credentials, new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()));
} else {
storageAccount = CloudStorageAccount.parse(storageConnectionString);
}
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
ResultContinuation nextContainerResults = null;
do {
try {
ResultSegment<CloudBlobContainer> contSegment = blobClient.listContainersSegmented(null, ContainerListingDetails.NONE, getQueryResultLimit(), nextContainerResults, null, null);
context.apiListStorageContainers++;
nextContainerResults = contSegment.getContinuationToken();
for (CloudBlobContainer container : contSegment.getResults()) {
String uri = canonizeId(container.getUri().toString());
context.containerIds.add(uri);
context.storageContainers.put(uri, container);
ResultContinuation nextBlobResults = null;
do {
ResultSegment<ListBlobItem> blobsSegment = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), getQueryResultLimit(), nextBlobResults, null, null);
context.apiListBlobs++;
nextBlobResults = blobsSegment.getContinuationToken();
for (ListBlobItem blobItem : blobsSegment.getResults()) {
String blobId = canonizeId(blobItem.getUri().toString());
context.storageBlobs.put(blobId, blobItem);
// populate mapping of blob uri and storage account for all storage
// accounts as new disks can be added to already existing blobs
StorageAccount blobStorageAccount = context.storageAccountMap.get(id);
if (blobStorageAccount != null) {
context.storageAccountBlobUriMap.put(blobId, blobStorageAccount);
}
context.blobIds.add(blobId);
}
} while (nextBlobResults != null);
}
} catch (StorageException storageExc) {
if (storageExc.getCause() instanceof UnknownHostException || StorageErrorCode.RESOURCE_NOT_FOUND.toString().equals(storageExc.getErrorCode()) || AzureConstants.RESOURCE_NOT_FOUND.equals(storageExc.getErrorCode())) {
String msg = "Probably trying to process a storage account/container that was " + "just deleted. Skipping it and continue with the next " + "storage account. Storage account id: [" + id + "], " + "storage account connection string: [" + storageConnectionString + "]. Error: %s";
logInfo(msg, Utils.toString(storageExc));
} else {
logSevere("StorageException[errorCode=%s, httpCode=%s, msg=%s, cause=%s]", storageExc.getErrorCode(), storageExc.getHttpStatusCode(), storageExc.getMessage(), storageExc.getCause() != null ? Utils.toString(storageExc.getCause()) : "n/a");
throw storageExc;
}
}
} while (nextContainerResults != null);
logFine(() -> String.format("Processing %d storage containers", context.containerIds.size()));
} catch (Exception e) {
handleError(context, e);
return;
}
}
context.subStage = next;
handleSubStage(context);
}, failure);
}
use of com.microsoft.azure.management.Azure in project photon-model by vmware.
the class AzureStorageEnumerationAdapterService method disassociateStorageDescription.
/*
* Disassociate local storage accounts and all resources inside them that no longer exist in
* Azure
*
* The logic works by recording a timestamp when enumeration starts. This timestamp is used to
* lookup resources which haven't been touched as part of current enumeration cycle. The other
* data point this method uses is the storage accounts discovered as part of get storage
* accounts call.
*
* A disassociate on a storage description is invoked only if it meets two criteria: -
* Timestamp older
* than current enumeration cycle. - Storage account is not present on Azure.
*/
private void disassociateStorageDescription(StorageEnumContext context, StorageEnumStages next) {
Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(StorageDescription.class).addRangeClause(StorageDescription.FIELD_NAME_UPDATE_TIME_MICROS, QueryTask.NumericRange.createLessThanRange(context.enumerationStartTimeInMicros));
QueryByPages<StorageDescription> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), StorageDescription.class, context.parentCompute.tenantLinks, context.request.endpointLink, context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
List<DeferredResult<Operation>> disassociateDRs = new ArrayList<>();
queryLocalStates.queryDocuments(sd -> {
if (context.storageAccountIds.contains(sd.id)) {
return;
}
// Deleting the localResourceState is done by disassociating the
// endpointLink from the localResourceState. If the localResourceState
// isn't associated with any other endpointLink, it should be eventually
// deleted by the groomer task
Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.endpointLink, sd);
if (disassociateOp == null) {
return;
}
// NOTE: The original Op is set with completion that must be executed.
// Since sendWithDeferredResult is used we must manually call it, otherwise it's
// just ignored.
CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
DeferredResult<Operation> disassociateDR = sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
final String message = "Disassociate stale %s state";
if (e != null) {
logWarning(message + ": FAILED with %s", sd.documentSelfLink, Utils.toString(e));
} else {
log(Level.FINEST, message + ": SUCCESS", sd.documentSelfLink);
}
});
disassociateDRs.add(disassociateDR);
}).thenCompose(ignore -> DeferredResult.allOf(disassociateDRs)).whenComplete((r, e) -> {
logFine(() -> "Finished disassociation of storage descriptions for Azure");
context.subStage = next;
handleSubStage(context);
});
}
use of com.microsoft.azure.management.Azure in project photon-model by vmware.
the class AzureStorageEnumerationAdapterService method disassociateResourceGroupStates.
/*
* Disassociate local storage containers that no longer exist in Azure
*
* The logic works by recording a timestamp when enumeration starts. This timestamp is used to
* lookup resources which haven't been touched as part of current enumeration cycle. The other
* data point this method uses is the storage accounts discovered as part of get storage
* accounts call.
*
* A disassociate on a resource group state is invoked only if it meets two criteria: -
* Timestamp
* older than current enumeration cycle. - Storage container is not present on Azure.
*/
private void disassociateResourceGroupStates(StorageEnumContext context, StorageEnumStages next) {
Query.Builder qBuilder = newResourceGroupQueryBuilder(context).addRangeClause(ResourceGroupState.FIELD_NAME_UPDATE_TIME_MICROS, QueryTask.NumericRange.createLessThanRange(context.enumerationStartTimeInMicros));
QueryByPages<ResourceGroupState> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), ResourceGroupState.class, context.parentCompute.tenantLinks, null, /* endpointLink */
context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
List<DeferredResult<Operation>> ops = new ArrayList<>();
queryLocalStates.queryDocuments(rg -> {
// repository it means nothing has changed about it.
if (context.containerIds.contains(canonizeId(rg.id))) {
return;
}
// Deleting the localResourceState is done by disassociating the
// endpointLink from the localResourceState. If the localResourceState
// isn't associated with any other endpointLink, it should be eventually
// deleted by the groomer task
Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.endpointLink, rg);
if (disassociateOp == null) {
return;
}
// NOTE: The original Op is set with completion that must be executed.
// Since sendWithDeferredResult is used we must manually call it, otherwise it's
// just ignored.
CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
DeferredResult<Operation> disassociateDR = sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
final String message = "Disassociate stale %s state";
if (e != null) {
logWarning(message + ": FAILED with %s", rg.documentSelfLink, Utils.toString(e));
} else {
log(Level.FINEST, message + ": SUCCESS", rg.documentSelfLink);
}
});
ops.add(disassociateDR);
}).thenCompose(r -> DeferredResult.allOf(ops)).whenComplete((r, e) -> {
logFine(() -> "Finished disassociation of storage containers for Azure");
context.subStage = next;
handleSubStage(context);
});
}
use of com.microsoft.azure.management.Azure in project photon-model by vmware.
the class AzureStorageEnumerationAdapterService method patchAdditionalFields.
/**
* Patch additional values to storage description fetched from azure's storage account.
*/
private void patchAdditionalFields(StorageEnumContext context, StorageEnumStages next) {
if (context.storageDescriptionsForPatching.size() == 0) {
logFine(() -> "No storage accounts need to be patched.");
context.subStage = next;
handleSubStage(context);
return;
}
// Patching power state and network Information. Hence 2.
// If we patch more fields, this number should be increased accordingly.
Consumer<Throwable> failure = e -> {
logWarning("Failure getting Azure storage accounts [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
handleError(context, e);
};
PhotonModelUtils.runInExecutor(this.executorService, () -> {
Azure azureClient = context.azureSdkClients.getAzureClient();
StorageAccountsInner stOps = azureClient.storageAccounts().inner();
DeferredResult.allOf(context.storageDescriptionsForPatching.values().stream().map(storageDescription -> patchStorageDetails(stOps, storageDescription)).map(dr -> dr.thenCompose(storageDescription -> sendWithDeferredResult(Operation.createPatch(context.request.buildUri(storageDescription.documentSelfLink)).setBody(storageDescription).setCompletion((o, e) -> {
if (e != null) {
logWarning(() -> String.format("Error updating Storage:[%s], reason: %s", storageDescription.name, e));
}
})))).collect(Collectors.toList())).whenComplete((all, e) -> {
if (e != null) {
logWarning(() -> String.format("Error: %s", e));
}
context.subStage = next;
handleSubStage(context);
});
}, failure);
}
Aggregations