Examples with DiskState com.vmware.photon.controller.model.resources.DiskService.DiskState used on opensource projects

Example 36 with DiskState

use of com.vmware.photon.controller.model.resources.DiskService.DiskState in project photon-model by vmware.

the class AWSComputeDiskDay2ServiceTest method createAndAttachExternalDisks.

private ComputeState createAndAttachExternalDisks(Instance instance, List<String> existingNames, ComputeState vmStateBeforeAttach, List<String> externallyProvisionedDisks, int numDisks) throws Throwable {
    for (int i = 2, j = 1; i < numDisks + 2; i++, j++) {
        // create disk
        DiskState diskSpec = createAWSDiskState(this.host, this.endpointState, this.currentTestName.getMethodName() + "_disk" + i, !(j == numDisks), this.zoneId, regionId);
        provisionSingleDisk(diskSpec);
        assertEquals(1, diskSpec.endpointLinks.size());
        ServiceDocumentQueryResult diskQueryResult = ProvisioningUtils.queryDiskInstances(this.host, vmStateBeforeAttach.diskLinks.size() + 1);
        DiskState provisionedDisk = new Gson().fromJson(diskQueryResult.documents.get(diskSpec.documentSelfLink).toString(), DiskState.class);
        // assert that the disk is available
        assertEquals("disk status not matching", DiskService.DiskStatus.AVAILABLE, provisionedDisk.status);
        assertNotNull("Disk creation time cannot be empty", provisionedDisk.creationTimeMicros);
        // collect external disks into a list
        externallyProvisionedDisks.add(provisionedDisk.documentSelfLink);
        // attach disk to the vm.
        performDiskOperationAndVerify(vmStateBeforeAttach.documentSelfLink, Arrays.asList(externallyProvisionedDisks.get(i - 2)), ResourceOperation.ATTACH_DISK.operation, DiskTaskService.DiskTaskState.SubStage.FINISHED);
        ComputeState vmStateAfterAttach = this.host.getServiceState(null, ComputeState.class, UriUtils.buildUri(this.host, this.vmState.documentSelfLink));
        // assert the number of disklinks have increased by 1.
        assertEquals(vmStateBeforeAttach.diskLinks.size() + 1, vmStateAfterAttach.diskLinks.size());
        DiskState attachedDisk = this.host.getServiceState(null, DiskState.class, UriUtils.buildUri(this.host, provisionedDisk.documentSelfLink));
        // assert that the attached disk has one endpoint in its endpointLinks.
        assertEquals(1, attachedDisk.endpointLinks.size());
        // assert that the disk is attached.
        assertEquals("disk status not matching", DiskService.DiskStatus.ATTACHED, attachedDisk.status);
        // assert the used device name is same as expected returned by AWS Device name utility.
        assertDeviceName(instance, attachedDisk, existingNames);
        vmStateBeforeAttach = vmStateAfterAttach;
    }
    return vmStateBeforeAttach;
}

Example 37 with DiskState

use of com.vmware.photon.controller.model.resources.DiskService.DiskState in project photon-model by vmware.

the class AWSComputeDiskDay2ServiceTest method testDiskOperations.

/**
 * This test performs the following steps in sequence.
 * 1. provision a VM with one bootdisk, two new inline disks and one existing disk.
 * 2. Create three independent disks and then explicitly attach all of them to the VM
 * 3. Detach the first two disks that are explicitly attached.
 * 4. Delete VM(all the attached disks which are marked are not marked to persist will also be deleted).
 */
@Test
public void testDiskOperations() throws Throwable {
    Gson gson = new Gson();
    DiskState diskspec1 = createAWSDiskState(this.host, this.endpointState, this.currentTestName.getMethodName() + "_disk1", Boolean.FALSE, this.zoneId, regionId);
    // create a disk
    provisionSingleDisk(diskspec1);
    assertEquals(1, diskspec1.endpointLinks.size());
    // attach a disk while provisioning the vm
    provisionVMAndAttachDisk(this.zoneId, diskspec1.documentSelfLink, true);
    // check that the VM has been created
    ServiceDocumentQueryResult computeQueryResult = ProvisioningUtils.queryComputeInstances(this.host, 2);
    ComputeState vmStateAfterAttach1 = gson.fromJson(computeQueryResult.documents.get(this.vmState.documentSelfLink).toString(), ComputeState.class);
    Instance instance = null;
    if (!this.isMock) {
        List<Instance> instances = getAwsInstancesByIds(this.client, this.host, Collections.singletonList(vmStateAfterAttach1.id));
        instance = instances.get(0);
        ComputeState vm = this.host.getServiceState(null, ComputeState.class, UriUtils.buildUri(this.host, this.vmState.documentSelfLink));
        assertAndSetVMSecurityGroupsToBeDeleted(instance, vm);
        // verify that the disk is attached while provisioning the vm.
        DiskState attachedDisk1 = this.host.getServiceState(null, DiskState.class, UriUtils.buildUri(this.host, diskspec1.documentSelfLink));
        assertEquals("disk status not matching", DiskService.DiskStatus.ATTACHED, attachedDisk1.status);
        assertEquals(1, attachedDisk1.endpointLinks.size());
        this.volumeId = attachedDisk1.id;
    }
    ServiceDocumentQueryResult initialDiskQueryResult = ProvisioningUtils.queryDiskInstances(this.host, vmStateAfterAttach1.diskLinks.size());
    List<String> existingNames = new ArrayList<>();
    for (String diskLink : initialDiskQueryResult.documentLinks) {
        DiskState diskState = Utils.fromJson(initialDiskQueryResult.documents.get(diskLink), DiskState.class);
        existingNames.add(diskState.customProperties.get(AWSConstants.DEVICE_NAME));
    }
    ComputeState vmStateBeforeAttach = vmStateAfterAttach1;
    int numExternalDisks = 3;
    List<String> externallyProvisionedDisks = new ArrayList<>();
    ComputeState vmAfterExternalDiskAttach = createAndAttachExternalDisks(instance, existingNames, vmStateBeforeAttach, externallyProvisionedDisks, numExternalDisks);
    List<String> diskLinksToDetach = externallyProvisionedDisks.stream().filter(diskLink -> !diskLink.equals(externallyProvisionedDisks.get(numExternalDisks - 1))).collect(Collectors.toList());
    // detach disks from the vm and verify the details of the detached disks
    ComputeState vmStateAfterExplicitDetach = detachDiskAndVerify(vmAfterExternalDiskAttach, diskLinksToDetach, this.disksToCleanUp);
    assertEquals(5, vmStateAfterExplicitDetach.diskLinks.size());
    // On VM delete, two inline(Test_Volume_1 and Test_Volume_2), one non-inline external(*_disk1),
    // one external-disk should be deleted. Only one of the attached disks should be persisted.
    deleteVMAndVerifyDisks(vmStateAfterExplicitDetach.documentSelfLink, vmStateAfterExplicitDetach.diskLinks);
}

Example 38 with DiskState

use of com.vmware.photon.controller.model.resources.DiskService.DiskState in project photon-model by vmware.

the class AWSS3StorageEnumerationAdapterService method enumerateTags.

/**
 * Calls getBucketTaggingConfiguration() on every bucket to enumerate bucket tags.
 * getBucketTaggingConfiguration() method is region aware and only returns valid results when we
 * call it with the client region same as the S3 bucket region. Since S3 bucket region is not
 * available through API, when a new bucket is discovered, we try to get tags for it by calling
 * getBucketTaggingConfiguration() with client in every AWS region. We then store the client region
 * for which we received successful response as region in DiskState for that S3 bucket. This region
 * in DiskState is then used for any subsequent calls for enumerating tags.
 */
private void enumerateTags(S3StorageEnumerationContext aws, S3StorageEnumerationSubStage next) {
    OperationContext operationContext = OperationContext.getOperationContext();
    this.executorService.submit(() -> {
        OperationContext.restoreOperationContext(operationContext);
        List<DeferredResult<S3ClientHandler>> s3ClientsEnumeratePreviousBuckets = new ArrayList<>();
        // Get a client for that region and make a call to enumerate tags.
        for (Map.Entry<String, DiskState> entry : aws.diskStatesToBeUpdatedByBucketName.entrySet()) {
            // with valid region in subsequent enumeration runs.
            if (entry.getValue().regionId != null) {
                aws.regionsByBucketName.put(entry.getValue().id, entry.getValue().regionId);
            } else {
                logWarning("Null region found in S3 diskState");
                Operation.createDelete(aws.service.getHost(), entry.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(getDeletionState(Utils.getNowMicrosUtc())).setCompletion((o, e) -> {
                    if (e != null) {
                        logWarning("Exception deleting diskState with null " + "region [ex=%s]", e.getMessage());
                        return;
                    }
                    logWarning("Deleted diskState with null region [diskState=%s]", Utils.toJsonHtml(entry.getValue()));
                }).sendWith(aws.service);
                continue;
            }
            s3ClientsEnumeratePreviousBuckets.add(aws.clientManager.getOrCreateS3ClientAsync(aws.endpointAuth, entry.getValue().regionId, aws.service).thenApply(s3Client -> {
                S3ClientHandler s3ClientHandler = new S3ClientHandler();
                s3ClientHandler.s3Client = s3Client;
                s3ClientHandler.bucketName = entry.getKey();
                s3ClientHandler.diskState = entry.getValue();
                return s3ClientHandler;
            }));
        }
        // Handler to enumerate new buckets not previously enumerated.
        BiConsumer<List<S3ClientHandler>, Throwable> enumerateNewBuckets = (s3ClientHandlers, throwable) -> {
            OperationContext.restoreOperationContext(operationContext);
            for (Bucket bucket : aws.bucketsToBeCreated) {
                for (S3ClientHandler s3ClientHandler : s3ClientHandlers) {
                    try {
                        BucketTaggingConfiguration bucketTaggingConfiguration = s3ClientHandler.s3Client.getBucketTaggingConfiguration(bucket.getName());
                        aws.regionsByBucketName.put(bucket.getName(), s3ClientHandler.region.getName());
                        if (bucketTaggingConfiguration != null) {
                            aws.tagsByBucketName.put(bucket.getName(), new ConcurrentHashMap<>());
                            bucketTaggingConfiguration.getAllTagSets().forEach(tagSet -> aws.tagsByBucketName.get(bucket.getName()).putAll(tagSet.getAllTags()));
                        }
                        break;
                    } catch (Exception e) {
                        // getbucketTaggingConfiguration().
                        if (e instanceof AmazonS3Exception && (((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_MOVED_PERM || ((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_FORBIDDEN || ((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_BAD_REQUEST)) {
                            continue;
                        } else {
                            logSevere("Exception enumerating tags for S3 bucket with unknown region " + "[endpoint=%s] [region=%s] [ex=%s]", aws.request.original.endpointLink, s3ClientHandler.region.getName(), e.getMessage());
                            continue;
                        }
                    }
                }
            }
            // Once completed, move to next stage.
            aws.subStage = next;
            handleReceivedEnumerationData(aws);
        };
        // Handler to enumerate tags for buckets already previously enumerated.
        BiConsumer<List<S3ClientHandler>, Throwable> enumeratePreviousBuckets = (s3ClientHandlers, ignored) -> {
            OperationContext.restoreOperationContext(operationContext);
            for (S3ClientHandler s3ClientHandler : s3ClientHandlers) {
                try {
                    BucketTaggingConfiguration bucketTaggingConfiguration = s3ClientHandler.s3Client.getBucketTaggingConfiguration(s3ClientHandler.bucketName);
                    if (bucketTaggingConfiguration != null) {
                        aws.tagsByBucketName.put(s3ClientHandler.bucketName, new ConcurrentHashMap<>());
                        bucketTaggingConfiguration.getAllTagSets().forEach(tagSet -> aws.tagsByBucketName.get(s3ClientHandler.bucketName).putAll(tagSet.getAllTags()));
                    }
                } catch (Exception e) {
                    logSevere("Exception enumerating tags for S3 bucket with known region " + "[endpoint=%s] [bucketName=%s - %s] [region=%s] [ex=%s]", aws.request.original.endpointLink, s3ClientHandler.bucketName, s3ClientHandler.diskState.id, s3ClientHandler.diskState.regionId, e.getMessage());
                }
            }
            // For remaining buckets, they have yet to be enumerated. Brute force and try to
            // enumerate tags for these buckets over every region until we find the correct
            // region and then store it in DiskState for future reference.
            List<DeferredResult<S3ClientHandler>> s3ClientBucketsToEnumerate = new ArrayList<>();
            for (Regions region : Regions.values()) {
                // Get an s3 client in the region asynchronously. Once all completed, these
                // clients will be used to test each bucket.
                s3ClientBucketsToEnumerate.add(aws.clientManager.getOrCreateS3ClientAsync(aws.endpointAuth, region.getName(), aws.service).thenApply(s3Client -> {
                    S3ClientHandler s3ClientHandler = new S3ClientHandler();
                    s3ClientHandler.s3Client = s3Client;
                    s3ClientHandler.region = region;
                    return s3ClientHandler;
                }));
            }
            // Once finished, attempt to enumerate each of the "new" buckets.
            DeferredResult.allOf(s3ClientBucketsToEnumerate).whenComplete(enumerateNewBuckets);
        };
        // Enumerate tags of previously enumerated buckets first
        DeferredResult.allOf(s3ClientsEnumeratePreviousBuckets).whenComplete(enumeratePreviousBuckets);
    });
}

Example 39 with DiskState

use of com.vmware.photon.controller.model.resources.DiskService.DiskState in project photon-model by vmware.

the class AWSS3StorageEnumerationAdapterService method getLocalResources.

/**
 * Query and get list of S3 buckets present locally in disk states in current context.
 */
private void getLocalResources(S3StorageEnumerationContext aws, S3StorageEnumerationSubStage next) {
    // instance Ids. the filtering is performed on the selected resource pool.
    if (aws.localResourcesNextPageLink == null) {
        Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(DiskState.class).addFieldClause(DiskState.FIELD_NAME_STORAGE_TYPE, STORAGE_TYPE_S3).addInClause(DiskState.FIELD_NAME_ID, aws.remoteBucketsByBucketName.keySet());
        addScopeCriteria(qBuilder, aws);
        QueryTask queryTask = QueryTask.Builder.createDirectTask().setQuery(qBuilder.build()).addOption(QueryOption.EXPAND_CONTENT).setResultLimit(getQueryResultLimit()).build();
        queryTask.tenantLinks = aws.parentCompute.tenantLinks;
        QueryUtils.startInventoryQueryTask(this, queryTask).whenComplete((qrt, e) -> {
            if (e != null) {
                this.logSevere(() -> String.format("Failure retrieving query" + " results: %s", e.toString()));
                signalErrorToEnumerationAdapter(aws, e);
                return;
            }
            qrt.results.documents.values().forEach(documentJson -> {
                DiskState localDisk = Utils.fromJson(documentJson, DiskState.class);
                aws.localDiskStatesByBucketName.put(localDisk.name, localDisk);
            });
            this.logFine(() -> String.format("%d S3 disk states found.", qrt.results.documentCount));
            if (qrt.results.nextPageLink != null) {
                this.logFine("Processing next page for local disk states.");
                aws.localResourcesNextPageLink = qrt.results.nextPageLink;
                handleReceivedEnumerationData(aws);
            } else {
                aws.subStage = next;
                handleReceivedEnumerationData(aws);
            }
        });
    } else {
        Operation.createGet(createInventoryUri(this.getHost(), aws.localResourcesNextPageLink)).setReferer(this.getUri()).setCompletion((o, e) -> {
            if (e != null) {
                this.logSevere(() -> String.format("Failure retrieving query" + " results: %s", e.toString()));
                signalErrorToEnumerationAdapter(aws, e);
                return;
            }
            QueryTask qrt = o.getBody(QueryTask.class);
            qrt.results.documents.values().forEach(documentJson -> {
                DiskState localDisk = Utils.fromJson(documentJson, DiskState.class);
                aws.localDiskStatesByBucketName.put(localDisk.name, localDisk);
            });
            this.logFine(() -> String.format("%d S3 disk states found.", qrt.results.documentCount));
            if (qrt.results.nextPageLink != null) {
                this.logFine("Processing next page for local disk states.");
                aws.localResourcesNextPageLink = qrt.results.nextPageLink;
                handleReceivedEnumerationData(aws);
            } else {
                aws.subStage = next;
                handleReceivedEnumerationData(aws);
            }
        }).sendWith(this);
    }
}

Example 40 with DiskState

use of com.vmware.photon.controller.model.resources.DiskService.DiskState in project photon-model by vmware.

the class AWSS3StorageEnumerationAdapterService method createDiskStates.

/**
 * Creates the disk states that represent the buckets received from AWS during
 * enumeration. Fields currently being enumerated for S3 are all immutable on AWS side, hence we only create
 * disks and don't patch to them in subsequent except for changes in tagLinks.
 */
private void createDiskStates(S3StorageEnumerationContext aws, S3StorageEnumerationSubStage next) {
    // For all the disks to be created, we filter them based on whether we were able to find the correct
    // region for the disk using getBucketTaggingConfiguration() call and then map them and create operations.
    // Filtering is done to avoid creating disk states with null region (since we don't PATCH region field
    // after creating the disk, we need to ensure that disk state is initially created with the correct region).
    // kick off the operation using a JOIN
    List<DiskState> diskStatesToBeCreated = new ArrayList<>();
    aws.bucketsToBeCreated.stream().filter(bucket -> aws.regionsByBucketName.containsKey(bucket.getName())).forEach(bucket -> {
        diskStatesToBeCreated.add(mapBucketToDiskState(bucket, aws));
    });
    diskStatesToBeCreated.forEach(diskState -> aws.enumerationOperations.add(createPostOperation(this, diskState, DiskService.FACTORY_LINK)));
    this.logFine(() -> String.format("Creating %d S3 disks", aws.bucketsToBeCreated.size()));
    // For those disk states which do not have the tagLink, add the tagLink by PATCHing those states.
    if (aws.internalTypeTagSelfLink != null) {
        aws.diskStatesToBeUpdatedByBucketName.entrySet().stream().filter(diskMap -> diskMap.getValue().tagLinks == null || !diskMap.getValue().tagLinks.contains(aws.internalTypeTagSelfLink)).forEach(diskMap -> {
            Map<String, Collection<Object>> collectionsToAddMap = Collections.singletonMap(DiskState.FIELD_NAME_TAG_LINKS, Collections.singletonList(aws.internalTypeTagSelfLink));
            Map<String, Collection<Object>> collectionsToRemoveMap = Collections.singletonMap(DiskState.FIELD_NAME_TAG_LINKS, Collections.emptyList());
            ServiceStateCollectionUpdateRequest updateTagLinksRequest = ServiceStateCollectionUpdateRequest.create(collectionsToAddMap, collectionsToRemoveMap);
            aws.enumerationOperations.add(Operation.createPatch(this.getHost(), diskMap.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(updateTagLinksRequest));
        });
    }
    // update endpointLinks
    aws.diskStatesToBeUpdatedByBucketName.entrySet().stream().filter(diskMap -> diskMap.getValue().endpointLinks == null || !diskMap.getValue().endpointLinks.contains(aws.request.original.endpointLink)).forEach(diskMap -> {
        Map<String, Collection<Object>> collectionsToAddMap = Collections.singletonMap(DiskState.FIELD_NAME_ENDPOINT_LINKS, Collections.singletonList(aws.request.original.endpointLink));
        Map<String, Collection<Object>> collectionsToRemoveMap = Collections.singletonMap(DiskState.FIELD_NAME_ENDPOINT_LINKS, Collections.emptyList());
        ServiceStateCollectionUpdateRequest updateEndpointLinksRequest = ServiceStateCollectionUpdateRequest.create(collectionsToAddMap, collectionsToRemoveMap);
        aws.enumerationOperations.add(Operation.createPatch(this.getHost(), diskMap.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(updateEndpointLinksRequest));
    });
    OperationJoin.JoinedCompletionHandler joinCompletion = (ox, exc) -> {
        if (exc != null) {
            this.logSevere(() -> String.format("Error creating/updating disk %s", Utils.toString(exc)));
            aws.subStage = S3StorageEnumerationSubStage.DELETE_DISKS;
            handleReceivedEnumerationData(aws);
            return;
        }
        ox.entrySet().stream().forEach(operationEntry -> {
            aws.diskStatesEnumerated.add(operationEntry.getValue().getBody(DiskState.class));
        });
        this.logFine(() -> "Successfully created and updated all the disk states.");
        aws.subStage = next;
        handleReceivedEnumerationData(aws);
    };
    if (aws.enumerationOperations.isEmpty()) {
        aws.subStage = next;
        handleReceivedEnumerationData(aws);
        return;
    }
    OperationJoin joinOp = OperationJoin.create(aws.enumerationOperations);
    joinOp.setCompletion(joinCompletion);
    joinOp.sendWith(this.getHost());
}