Examples with StorageAccount com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount used on opensource projects

Example 1 with StorageAccount

use of com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method createStorageDescription.

private DeferredResult<StorageDescription> createStorageDescription(StorageEnumContext context, StorageAccount storageAccount, StorageAccountsInner stOps) {
    String resourceGroupName = getResourceGroupName(storageAccount.id);
    AzureDeferredResultServiceCallback<StorageAccountListKeysResultInner> handler = new Default<>(this, "Load account keys for storage: " + storageAccount.name);
    PhotonModelUtils.runInExecutor(this.executorService, () -> {
        stOps.listKeysAsync(resourceGroupName, storageAccount.name, handler);
    }, handler::failure);
    return handler.toDeferredResult().thenCompose(keys -> AzureUtils.storeKeys(getHost(), keys, context.request.endpointLink, context.parentCompute.tenantLinks)).thenApply(auth -> {
        String connectionString = String.format(STORAGE_CONNECTION_STRING, storageAccount.name, auth.customProperties.get(AZURE_STORAGE_ACCOUNT_KEY1));
        context.storageConnectionStrings.put(storageAccount.id, connectionString);
        return auth;
    }).thenApply(auth -> {
        StorageDescription storageDesc = AzureUtils.constructStorageDescription(context.parentCompute, context.request, storageAccount, auth.documentSelfLink);
        return storageDesc;
    }).thenCompose(sd -> sendWithDeferredResult(Operation.createPost(context.request.buildUri(StorageDescriptionService.FACTORY_LINK)).setBody(sd).setCompletion((o, e) -> {
        if (e != null) {
            logWarning("Unable to store storage description for storage account:[%s], reason: %s", storageAccount.name, Utils.toJsonHtml(e));
        } else {
            StorageDescription storageDescription = o.getBody(StorageDescription.class);
            context.storageDescriptionsForPatching.put(storageDescription.id, storageDescription);
        }
    }), StorageDescription.class));
}

Example 2 with StorageAccount

use of com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method getStorageContainersAsync.

/*
     * Get all Azure containers by storage account
     */
public void getStorageContainersAsync(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageAccountIds.size() == 0) {
        logFine(() -> "No storage description available - clean up all resources");
        context.subStage = StorageEnumStages.DISASSOCIATE_RESOURCE_GROUP_STATES;
        handleSubStage(context);
        return;
    }
    Consumer<Throwable> failure = e -> {
        logWarning("Failure getting Azure storage containers [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
        handleError(context, e);
    };
    PhotonModelUtils.runInExecutor(this.executorService, () -> {
        for (String id : context.storageAccountIds) {
            String storageConnectionString = context.storageConnectionStrings.get(id);
            if (storageConnectionString == null) {
                continue;
            }
            try {
                CloudStorageAccount storageAccount;
                if (AzureUtils.isAzureClientMock()) {
                    StorageCredentials credentials = StorageCredentials.tryParseCredentials(storageConnectionString);
                    storageAccount = new CloudStorageAccount(credentials, new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()));
                } else {
                    storageAccount = CloudStorageAccount.parse(storageConnectionString);
                }
                CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
                ResultContinuation nextContainerResults = null;
                do {
                    try {
                        ResultSegment<CloudBlobContainer> contSegment = blobClient.listContainersSegmented(null, ContainerListingDetails.NONE, getQueryResultLimit(), nextContainerResults, null, null);
                        context.apiListStorageContainers++;
                        nextContainerResults = contSegment.getContinuationToken();
                        for (CloudBlobContainer container : contSegment.getResults()) {
                            String uri = canonizeId(container.getUri().toString());
                            context.containerIds.add(uri);
                            context.storageContainers.put(uri, container);
                            ResultContinuation nextBlobResults = null;
                            do {
                                ResultSegment<ListBlobItem> blobsSegment = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), getQueryResultLimit(), nextBlobResults, null, null);
                                context.apiListBlobs++;
                                nextBlobResults = blobsSegment.getContinuationToken();
                                for (ListBlobItem blobItem : blobsSegment.getResults()) {
                                    String blobId = canonizeId(blobItem.getUri().toString());
                                    context.storageBlobs.put(blobId, blobItem);
                                    // populate mapping of blob uri and storage account for all storage
                                    // accounts as new disks can be added to already existing blobs
                                    StorageAccount blobStorageAccount = context.storageAccountMap.get(id);
                                    if (blobStorageAccount != null) {
                                        context.storageAccountBlobUriMap.put(blobId, blobStorageAccount);
                                    }
                                    context.blobIds.add(blobId);
                                }
                            } while (nextBlobResults != null);
                        }
                    } catch (StorageException storageExc) {
                        if (storageExc.getCause() instanceof UnknownHostException || StorageErrorCode.RESOURCE_NOT_FOUND.toString().equals(storageExc.getErrorCode()) || AzureConstants.RESOURCE_NOT_FOUND.equals(storageExc.getErrorCode())) {
                            String msg = "Probably trying to process a storage account/container that was " + "just deleted. Skipping it and continue with the next " + "storage account. Storage account id: [" + id + "], " + "storage account connection string: [" + storageConnectionString + "]. Error: %s";
                            logInfo(msg, Utils.toString(storageExc));
                        } else {
                            logSevere("StorageException[errorCode=%s, httpCode=%s, msg=%s, cause=%s]", storageExc.getErrorCode(), storageExc.getHttpStatusCode(), storageExc.getMessage(), storageExc.getCause() != null ? Utils.toString(storageExc.getCause()) : "n/a");
                            throw storageExc;
                        }
                    }
                } while (nextContainerResults != null);
                logFine(() -> String.format("Processing %d storage containers", context.containerIds.size()));
            } catch (Exception e) {
                handleError(context, e);
                return;
            }
        }
        context.subStage = next;
        handleSubStage(context);
    }, failure);
}

Example 3 with StorageAccount

use of com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method createDiskStateObject.

private DiskState createDiskStateObject(StorageEnumContext context, ListBlobItem blob, String containerLink, DiskState oldDiskState) {
    DiskState diskState = new DiskState();
    diskState.name = getAzureBlobName(blob);
    if (containerLink != null) {
        diskState.storageDescriptionLink = containerLink;
    }
    diskState.resourcePoolLink = context.request.resourcePoolLink;
    diskState.computeHostLink = context.parentCompute.documentSelfLink;
    diskState.endpointLink = context.request.endpointLink;
    if (oldDiskState != null) {
        // first copy the existing ones
        diskState.endpointLinks = oldDiskState.endpointLinks;
    }
    // then append new one
    AdapterUtils.addToEndpointLinks(diskState, context.request.endpointLink);
    diskState.tenantLinks = context.parentCompute.tenantLinks;
    long bLength = 0;
    if (blob instanceof CloudBlob) {
        CloudBlob blobItem = (CloudBlob) blob;
        bLength = blobItem.getProperties().getLength();
    }
    diskState.status = DiskService.DiskStatus.AVAILABLE;
    diskState.capacityMBytes = bLength / B_TO_MB_FACTOR;
    diskState.customProperties = new HashMap<>();
    String storageType = isDisk(diskState.name) ? AZURE_STORAGE_DISKS : AZURE_STORAGE_BLOBS;
    diskState.storageType = storageType;
    if (oldDiskState != null) {
        diskState.tagLinks = (oldDiskState.tagLinks == null) ? new HashSet<>() : oldDiskState.tagLinks;
    } else {
        diskState.tagLinks = new HashSet<>();
    }
    switch(storageType) {
        case AZURE_STORAGE_DISKS:
            if (context.internalVhdTypeTag != null) {
                diskState.tagLinks.add(context.internalVhdTypeTag);
            }
            break;
        case AZURE_STORAGE_BLOBS:
            if (context.internalVhdTypeTag != null) {
                diskState.tagLinks.add(context.internalBlobTypeTag);
            }
            break;
        default:
            break;
    }
    // following two properties are set to defaults - can't retrieve that information from
    // existing calls
    diskState.type = DEFAULT_DISK_TYPE;
    if (oldDiskState != null) {
        if (diskState.storageDescriptionLink == null) {
            diskState.storageDescriptionLink = oldDiskState.storageDescriptionLink;
        }
        diskState.id = oldDiskState.id;
        diskState.documentSelfLink = oldDiskState.documentSelfLink;
        diskState.regionId = oldDiskState.regionId;
    } else {
        diskState.id = canonizeId(blob.getUri().toString());
        StorageAccount storageAccount = context.storageAccountBlobUriMap.get(diskState.id);
        if (storageAccount != null) {
            diskState.regionId = storageAccount.location;
        }
        diskState.documentSelfLink = UUID.randomUUID().toString();
    }
    return diskState;
}

Example 4 with StorageAccount

use of com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method getLocalStorageAccountDescriptions.

/**
 * Query all storage descriptions for the cluster filtered by the received set of storage
 * account Ids
 */
private void getLocalStorageAccountDescriptions(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageAccountsToUpdateCreate.isEmpty()) {
        context.subStage = StorageEnumStages.CREATE_STORAGE_DESCRIPTIONS;
        handleSubStage(context);
        return;
    }
    context.storageDescriptions.clear();
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(StorageDescription.class);
    Query.Builder instanceIdFilterParentQuery = Query.Builder.create(Occurance.MUST_OCCUR);
    for (Map.Entry<String, StorageAccount> account : context.storageAccountsToUpdateCreate.entrySet()) {
        Query instanceIdFilter = Query.Builder.create(Occurance.SHOULD_OCCUR).addFieldClause(StorageDescription.FIELD_NAME_ID, canonizeId(account.getValue().id)).build();
        instanceIdFilterParentQuery.addClause(instanceIdFilter);
    }
    qBuilder.addClause(instanceIdFilterParentQuery.build());
    QueryByPages<StorageDescription> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), StorageDescription.class, context.parentCompute.tenantLinks, null, /* endpointLink */
    context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    queryLocalStates.collectDocuments(Collectors.toList()).whenComplete((sds, ex) -> {
        if (ex != null) {
            handleError(context, ex);
            return;
        }
        logFine(() -> String.format("Found %d matching storage descriptions for Azure" + " storage accounts", sds.size()));
        List<DeferredResult<AuthCredentialsServiceState>> results = sds.stream().map(sd -> {
            context.storageDescriptions.put(sd.id, sd);
            // populate connectionStrings
            if (!context.storageConnectionStrings.containsKey(sd.id)) {
                return loadStorageAuth(context, sd);
            } else {
                return DeferredResult.<AuthCredentialsServiceState>completed(null);
            }
        }).collect(Collectors.toList());
        DeferredResult.allOf(results).whenComplete((creds, e) -> {
            if (e != null) {
                logWarning(() -> String.format("Failed to get storage description" + " credentials: %s", e.getMessage()));
            }
            context.subStage = next;
            handleSubStage(context);
        });
    });
}

Example 5 with StorageAccount

use of com.vmware.photon.controller.model.adapters.azure.model.storage.StorageAccount in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method updateStorageDescriptions.

/**
 * Updates matching storage descriptions for given storage accounts.
 */
private void updateStorageDescriptions(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageDescriptions.size() == 0) {
        logFine(() -> "No storage descriptions available for update");
        context.subStage = next;
        handleSubStage(context);
        return;
    }
    List<DeferredResult<Operation>> updates = context.storageDescriptions.values().stream().map(sd -> {
        StorageAccount storageAccount = context.storageAccountsToUpdateCreate.remove(sd.id);
        StorageDescription storageDescriptionToUpdate = new StorageDescription();
        storageDescriptionToUpdate.name = storageAccount.name;
        storageDescriptionToUpdate.authCredentialsLink = sd.authCredentialsLink;
        storageDescriptionToUpdate.regionId = storageAccount.location;
        storageDescriptionToUpdate.documentSelfLink = sd.documentSelfLink;
        if (StringUtils.isEmpty(storageDescriptionToUpdate.endpointLink)) {
            storageDescriptionToUpdate.endpointLink = sd.endpointLink;
        }
        storageDescriptionToUpdate.computeHostLink = sd.computeHostLink;
        // first copy
        storageDescriptionToUpdate.endpointLinks = sd.endpointLinks;
        // then update it
        AdapterUtils.addToEndpointLinks(storageDescriptionToUpdate, context.request.endpointLink);
        storageDescriptionToUpdate.tenantLinks = sd.tenantLinks;
        storageDescriptionToUpdate.regionId = storageAccount.location;
        // Check if SSE (encryption) is enable on azure storage account
        storageDescriptionToUpdate.supportsEncryption = storageAccount.properties.encryption != null ? storageAccount.properties.encryption.services.blob.enabled : false;
        // Set type of azure storage account
        storageDescriptionToUpdate.type = storageAccount.sku.name;
        context.storageDescriptionsForPatching.put(sd.id, sd);
        return storageDescriptionToUpdate;
    }).map(sd -> Operation.createPatch(this, sd.documentSelfLink).setBody(sd).setCompletion((o, e) -> {
        if (e != null) {
            logWarning(() -> String.format("Failed to update storage description:" + " %s", e.getMessage()));
        }
    })).map(o -> sendWithDeferredResult(o)).collect(java.util.stream.Collectors.toList());
    DeferredResult.allOf(updates).whenComplete((ignore, e) -> {
        logFine(() -> "Finished updating storage descriptions");
        context.subStage = next;
        handleSubStage(context);
    });
}