Search in sources :

Example 1 with ResultSegment

use of com.microsoft.azure.storage.ResultSegment in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method getStorageContainersAsync.

/*
     * Get all Azure containers by storage account
     */
public void getStorageContainersAsync(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageAccountIds.size() == 0) {
        logFine(() -> "No storage description available - clean up all resources");
        context.subStage = StorageEnumStages.DISASSOCIATE_RESOURCE_GROUP_STATES;
        handleSubStage(context);
        return;
    }
    Consumer<Throwable> failure = e -> {
        logWarning("Failure getting Azure storage containers [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
        handleError(context, e);
    };
    PhotonModelUtils.runInExecutor(this.executorService, () -> {
        for (String id : context.storageAccountIds) {
            String storageConnectionString = context.storageConnectionStrings.get(id);
            if (storageConnectionString == null) {
                continue;
            }
            try {
                CloudStorageAccount storageAccount;
                if (AzureUtils.isAzureClientMock()) {
                    StorageCredentials credentials = StorageCredentials.tryParseCredentials(storageConnectionString);
                    storageAccount = new CloudStorageAccount(credentials, new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()));
                } else {
                    storageAccount = CloudStorageAccount.parse(storageConnectionString);
                }
                CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
                ResultContinuation nextContainerResults = null;
                do {
                    try {
                        ResultSegment<CloudBlobContainer> contSegment = blobClient.listContainersSegmented(null, ContainerListingDetails.NONE, getQueryResultLimit(), nextContainerResults, null, null);
                        context.apiListStorageContainers++;
                        nextContainerResults = contSegment.getContinuationToken();
                        for (CloudBlobContainer container : contSegment.getResults()) {
                            String uri = canonizeId(container.getUri().toString());
                            context.containerIds.add(uri);
                            context.storageContainers.put(uri, container);
                            ResultContinuation nextBlobResults = null;
                            do {
                                ResultSegment<ListBlobItem> blobsSegment = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), getQueryResultLimit(), nextBlobResults, null, null);
                                context.apiListBlobs++;
                                nextBlobResults = blobsSegment.getContinuationToken();
                                for (ListBlobItem blobItem : blobsSegment.getResults()) {
                                    String blobId = canonizeId(blobItem.getUri().toString());
                                    context.storageBlobs.put(blobId, blobItem);
                                    // populate mapping of blob uri and storage account for all storage
                                    // accounts as new disks can be added to already existing blobs
                                    StorageAccount blobStorageAccount = context.storageAccountMap.get(id);
                                    if (blobStorageAccount != null) {
                                        context.storageAccountBlobUriMap.put(blobId, blobStorageAccount);
                                    }
                                    context.blobIds.add(blobId);
                                }
                            } while (nextBlobResults != null);
                        }
                    } catch (StorageException storageExc) {
                        if (storageExc.getCause() instanceof UnknownHostException || StorageErrorCode.RESOURCE_NOT_FOUND.toString().equals(storageExc.getErrorCode()) || AzureConstants.RESOURCE_NOT_FOUND.equals(storageExc.getErrorCode())) {
                            String msg = "Probably trying to process a storage account/container that was " + "just deleted. Skipping it and continue with the next " + "storage account. Storage account id: [" + id + "], " + "storage account connection string: [" + storageConnectionString + "]. Error: %s";
                            logInfo(msg, Utils.toString(storageExc));
                        } else {
                            logSevere("StorageException[errorCode=%s, httpCode=%s, msg=%s, cause=%s]", storageExc.getErrorCode(), storageExc.getHttpStatusCode(), storageExc.getMessage(), storageExc.getCause() != null ? Utils.toString(storageExc.getCause()) : "n/a");
                            throw storageExc;
                        }
                    }
                } while (nextContainerResults != null);
                logFine(() -> String.format("Processing %d storage containers", context.containerIds.size()));
            } catch (Exception e) {
                handleError(context, e);
                return;
            }
        }
        context.subStage = next;
        handleSubStage(context);
    }, failure);
}
Also used : STORAGE_ACCOUNT_REST_API_VERSION(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.STORAGE_ACCOUNT_REST_API_VERSION) Arrays(java.util.Arrays) QUERY_PARAM_API_VERSION(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.QUERY_PARAM_API_VERSION) ComputeEnumerateResourceRequest(com.vmware.photon.controller.model.adapterapi.ComputeEnumerateResourceRequest) AZURE_STORAGE_BLOBS(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_BLOBS) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) StringUtils(org.apache.commons.lang3.StringUtils) ResourceGroupState(com.vmware.photon.controller.model.resources.ResourceGroupService.ResourceGroupState) Azure(com.microsoft.azure.management.Azure) Utils(com.vmware.xenon.common.Utils) Map(java.util.Map) StorageDescription(com.vmware.photon.controller.model.resources.StorageDescriptionService.StorageDescription) COMPUTE_HOST_LINK_PROP_NAME(com.vmware.photon.controller.model.ComputeProperties.COMPUTE_HOST_LINK_PROP_NAME) EnumSet(java.util.EnumSet) ListBlobItem(com.microsoft.azure.storage.blob.ListBlobItem) AZURE_STORAGE_ACCOUNT_KEY1(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_ACCOUNT_KEY1) StorageAccountListKeysResultInner(com.microsoft.azure.management.storage.implementation.StorageAccountListKeysResultInner) StatelessService(com.vmware.xenon.common.StatelessService) Set(java.util.Set) AdapterUtils.getDeletionState(com.vmware.photon.controller.model.adapters.util.AdapterUtils.getDeletionState) AZURE_STORAGE_CONTAINER_LEASE_STATUS(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_CONTAINER_LEASE_STATUS) TagService(com.vmware.photon.controller.model.resources.TagService) StorageDescriptionService(com.vmware.photon.controller.model.resources.StorageDescriptionService) CompletionHandler(com.vmware.xenon.common.Operation.CompletionHandler) DeferredResult(com.vmware.xenon.common.DeferredResult) UriUtils(com.vmware.xenon.common.UriUtils) ComputeService(com.vmware.photon.controller.model.resources.ComputeService) AZURE_STORAGE_CONTAINER_LEASE_STATE(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_CONTAINER_LEASE_STATE) ComputeProperties(com.vmware.photon.controller.model.ComputeProperties) ResourceGroupStateType(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.ResourceGroupStateType) PhotonModelUtils(com.vmware.photon.controller.model.resources.util.PhotonModelUtils) ArrayList(java.util.ArrayList) StorageException(com.microsoft.azure.storage.StorageException) TagState(com.vmware.photon.controller.model.resources.TagService.TagState) Query(com.vmware.xenon.services.common.QueryTask.Query) AUTH_HEADER_BEARER_PREFIX(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AUTH_HEADER_BEARER_PREFIX) UriPaths(com.vmware.photon.controller.model.UriPaths) EnumerationStages(com.vmware.photon.controller.model.adapters.util.enums.EnumerationStages) CloudBlobContainer(com.microsoft.azure.storage.blob.CloudBlobContainer) StorageErrorCode(com.microsoft.azure.storage.StorageErrorCode) EnumUtils(org.apache.commons.lang3.EnumUtils) AZURE_STORAGE_CONTAINER_LEASE_LAST_MODIFIED(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_CONTAINER_LEASE_LAST_MODIFIED) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) LIST_STORAGE_ACCOUNTS(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.LIST_STORAGE_ACCOUNTS) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) ResultSegment(com.microsoft.azure.storage.ResultSegment) EMPTY_STR(com.vmware.photon.controller.model.constants.PhotonModelConstants.EMPTY_STR) ContainerListingDetails(com.microsoft.azure.storage.blob.ContainerListingDetails) UnknownHostException(java.net.UnknownHostException) QueryTop(com.vmware.photon.controller.model.query.QueryUtils.QueryTop) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) ComputeEnumerateAdapterRequest(com.vmware.photon.controller.model.adapters.util.ComputeEnumerateAdapterRequest) QuerySpecification(com.vmware.xenon.services.common.QueryTask.QuerySpecification) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) ResultContinuation(com.microsoft.azure.storage.ResultContinuation) URISyntaxException(java.net.URISyntaxException) QueryTask(com.vmware.xenon.services.common.QueryTask) AzureUriPaths(com.vmware.photon.controller.model.adapters.azure.AzureUriPaths) DEFAULT_DISK_TYPE(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.DEFAULT_DISK_TYPE) AzureSdkClients(com.vmware.photon.controller.model.adapters.azure.utils.AzureSdkClients) CUSTOM_PROP_ENDPOINT_LINK(com.vmware.photon.controller.model.constants.PhotonModelConstants.CUSTOM_PROP_ENDPOINT_LINK) URI(java.net.URI) TagsUtil.newTagState(com.vmware.photon.controller.model.adapters.util.TagsUtil.newTagState) AzureConstants(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants) AzureConstants.getQueryResultLimit(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.getQueryResultLimit) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) AZURE_STORAGE_CONTAINERS(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_CONTAINERS) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) Occurance(com.vmware.xenon.services.common.QueryTask.Query.Occurance) UUID(java.util.UUID) CloudStorageAccount(com.microsoft.azure.storage.CloudStorageAccount) Collectors(java.util.stream.Collectors) ResourceGroupService(com.vmware.photon.controller.model.resources.ResourceGroupService) List(java.util.List) AzureUtils(com.vmware.photon.controller.model.adapters.azure.utils.AzureUtils) AzureUtils.getResourceGroupName(com.vmware.photon.controller.model.adapters.azure.utils.AzureUtils.getResourceGroupName) TAG_KEY_TYPE(com.vmware.photon.controller.model.constants.PhotonModelConstants.TAG_KEY_TYPE) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) DiskService(com.vmware.photon.controller.model.resources.DiskService) Default(com.vmware.photon.controller.model.adapters.azure.utils.AzureDeferredResultServiceCallback.Default) QueryByPages(com.vmware.photon.controller.model.query.QueryUtils.QueryByPages) HashMap(java.util.HashMap) Level(java.util.logging.Level) HashSet(java.util.HashSet) AZURE_STORAGE_TYPE(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_TYPE) AuthCredentialsService(com.vmware.xenon.services.common.AuthCredentialsService) EnumerationAction(com.vmware.photon.controller.model.adapterapi.EnumerationAction) AzureResourceType(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AzureResourceType) STORAGE_CONNECTION_STRING(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.STORAGE_CONNECTION_STRING) ExecutorService(java.util.concurrent.ExecutorService) StorageAccountResultList(com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccountResultList) CloudBlobClient(com.microsoft.azure.storage.blob.CloudBlobClient) AdapterUriUtil(com.vmware.photon.controller.model.adapters.util.AdapterUriUtil) Operation(com.vmware.xenon.common.Operation) AZURE_STORAGE_DISKS(com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_STORAGE_DISKS) StorageAccountsInner(com.microsoft.azure.management.storage.implementation.StorageAccountsInner) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) StorageCredentials(com.microsoft.azure.storage.StorageCredentials) AzureUtils.canonizeId(com.vmware.photon.controller.model.adapters.azure.utils.AzureUtils.canonizeId) BlobListingDetails(com.microsoft.azure.storage.blob.BlobListingDetails) AzureDeferredResultServiceCallback(com.vmware.photon.controller.model.adapters.azure.utils.AzureDeferredResultServiceCallback) StorageAccountInner(com.microsoft.azure.management.storage.implementation.StorageAccountInner) StorageAccount(com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount) OperationJoin(com.vmware.xenon.common.OperationJoin) CloudBlob(com.microsoft.azure.storage.blob.CloudBlob) CloudBlobClient(com.microsoft.azure.storage.blob.CloudBlobClient) BlobListingDetails(com.microsoft.azure.storage.blob.BlobListingDetails) ListBlobItem(com.microsoft.azure.storage.blob.ListBlobItem) UnknownHostException(java.net.UnknownHostException) ResultContinuation(com.microsoft.azure.storage.ResultContinuation) CloudStorageAccount(com.microsoft.azure.storage.CloudStorageAccount) URI(java.net.URI) StorageException(com.microsoft.azure.storage.StorageException) UnknownHostException(java.net.UnknownHostException) URISyntaxException(java.net.URISyntaxException) StorageCredentials(com.microsoft.azure.storage.StorageCredentials) CloudStorageAccount(com.microsoft.azure.storage.CloudStorageAccount) StorageAccount(com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount) CloudBlobContainer(com.microsoft.azure.storage.blob.CloudBlobContainer) StorageException(com.microsoft.azure.storage.StorageException)

Aggregations

Azure (com.microsoft.azure.management.Azure)1 StorageAccountInner (com.microsoft.azure.management.storage.implementation.StorageAccountInner)1 StorageAccountListKeysResultInner (com.microsoft.azure.management.storage.implementation.StorageAccountListKeysResultInner)1 StorageAccountsInner (com.microsoft.azure.management.storage.implementation.StorageAccountsInner)1 CloudStorageAccount (com.microsoft.azure.storage.CloudStorageAccount)1 ResultContinuation (com.microsoft.azure.storage.ResultContinuation)1 ResultSegment (com.microsoft.azure.storage.ResultSegment)1 StorageCredentials (com.microsoft.azure.storage.StorageCredentials)1 StorageErrorCode (com.microsoft.azure.storage.StorageErrorCode)1 StorageException (com.microsoft.azure.storage.StorageException)1 BlobListingDetails (com.microsoft.azure.storage.blob.BlobListingDetails)1 CloudBlob (com.microsoft.azure.storage.blob.CloudBlob)1 CloudBlobClient (com.microsoft.azure.storage.blob.CloudBlobClient)1 CloudBlobContainer (com.microsoft.azure.storage.blob.CloudBlobContainer)1 ContainerListingDetails (com.microsoft.azure.storage.blob.ContainerListingDetails)1 ListBlobItem (com.microsoft.azure.storage.blob.ListBlobItem)1 ComputeProperties (com.vmware.photon.controller.model.ComputeProperties)1 COMPUTE_HOST_LINK_PROP_NAME (com.vmware.photon.controller.model.ComputeProperties.COMPUTE_HOST_LINK_PROP_NAME)1 UriPaths (com.vmware.photon.controller.model.UriPaths)1 ComputeEnumerateResourceRequest (com.vmware.photon.controller.model.adapterapi.ComputeEnumerateResourceRequest)1