Examples with ComputeState com.vmware.photon.controller.model.resources.ComputeService.ComputeState used on opensource projects

Example 71 with ComputeState

use of com.vmware.photon.controller.model.resources.ComputeService.ComputeState in project photon-model by vmware.

the class TestAzureStatsCollection method verifyStatsCollection.

/**
 * Verifies whether all the stats are being collected and whether last successful stats collection time
 * is more recent than the time from previous stats run.
 * @param selfLink
 */
private void verifyStatsCollection(String selfLink) {
    Operation computeStateOp = Operation.createGet(UriUtils.buildUri(this.host, selfLink)).setReferer(this.host.getUri()).setCompletion((o, e) -> {
        if (e != null) {
            this.host.log(Level.SEVERE, "Error getting compute state from document link");
        }
    });
    Operation response = this.host.waitForResponse(computeStateOp);
    ComputeState computeState = response.getBody(ComputeState.class);
    this.previousRunLastSuccessfulCollectionTimeInMicrosForHost = this.currentRunLastSuccessfulCollectionTimeInMicrosForHost;
    this.previousRunLastSuccessfulCollectionTimeInMicrosForVM = this.currentRunLastSuccessfulCollectionTimeInMicrosForVM;
    // Verify stats collection based on VM type.
    if (computeState.type == ComputeType.ENDPOINT_HOST) {
        this.resourceMetric = getResourceMetrics(selfLink, PhotonModelConstants.STORAGE_USED_BYTES);
        assertNotNull("No StorageUsedBytes metric present for host", this.resourceMetric);
        this.resourceMetric = getResourceMetrics(selfLink, PhotonModelConstants.MEMORY_USED_PERCENT);
        assertNotNull("No MemoryUsedPercent metric present for host", this.resourceMetric);
        this.resourceMetric = getResourceMetrics(selfLink, PhotonModelConstants.CPU_UTILIZATION_PERCENT);
        assertNotNull("No CpuUtilizationPercent metric present for host", this.resourceMetric);
        this.resourceMetric = getResourceMetrics(selfLink, LAST_COLLECTION_TIME_KEY_FOR_HOST);
        this.currentRunLastSuccessfulCollectionTimeInMicrosForHost = this.resourceMetric.entries.get(LAST_COLLECTION_TIME_KEY_FOR_HOST);
        assertTrue(this.currentRunLastSuccessfulCollectionTimeInMicrosForHost > this.previousRunLastSuccessfulCollectionTimeInMicrosForHost);
    } else if (computeState.type == ComputeType.VM_GUEST) {
        this.resourceMetric = getResourceMetrics(selfLink, PhotonModelConstants.MEMORY_USED_PERCENT);
        assertNotNull("No MemoryUsedPercent metric present for VM", this.resourceMetric);
        this.resourceMetric = getResourceMetrics(selfLink, PhotonModelConstants.CPU_UTILIZATION_PERCENT);
        assertNotNull("No CpuUtilizationPercent metric present for VM", this.resourceMetric);
        this.resourceMetric = getResourceMetrics(selfLink, LAST_COLLECTION_TIME_KEY_FOR_VM);
        this.currentRunLastSuccessfulCollectionTimeInMicrosForVM = this.resourceMetric.entries.get(LAST_COLLECTION_TIME_KEY_FOR_VM);
        assertTrue(this.currentRunLastSuccessfulCollectionTimeInMicrosForVM > this.previousRunLastSuccessfulCollectionTimeInMicrosForVM);
    } else {
        this.host.log(Level.SEVERE, "Unable to validate stats collection for this compute type");
    }
}

Example 72 with ComputeState

use of com.vmware.photon.controller.model.resources.ComputeService.ComputeState in project photon-model by vmware.

the class AzureSubscriptionsEnumerationServiceTest method testAddFirstAzureSubscription.

private void testAddFirstAzureSubscription() throws Throwable {
    // Fail if this.computeLink is null
    Assert.assertNotNull("Root computeLink of Azure EA account is null", this.computeLink);
    // Fail if this.compute is null
    Assert.assertNotNull("Root compute of Azure EA account is null", this.compute);
    // Request for creating computes for 1 Azure Subscriptions
    AzureSubscription subscription = getAzureSubscription(SUBSCRIPTION_ID_1, ACCOUNT_EMAIL_ID_1);
    AzureSubscription existingSubscription1 = getAzureSubscription(SUBSCRIPTION_EXISTING_1, ACCOUNT_EMAIL_ID_1);
    createAzureCostComputesForSubscriptions(Arrays.asList(subscription, existingSubscription1));
    // Query for Azure Computes created with CLIENT_ID as enrollment Number
    QueryTask task = createQueryTaskForAzureComputes(ENROLLMENT_NUMNBER, Collections.singletonList(TENANT_ID));
    QueryTask queryTaskResponse = executQuerySynchronously(task);
    assertQueryTaskResponse(queryTaskResponse, 3);
    // Remove compute for existing subscription
    ComputeState existingSubsCs = Utils.fromJson(queryTaskResponse.results.documents.remove(this.existingSubsComputeLink1), ComputeState.class);
    Assert.assertNotNull(existingSubsCs);
    Assert.assertNull(existingSubsCs.customProperties.get(PhotonModelConstants.AUTO_DISCOVERED_ENTITY));
    // Get and assert the returned compute
    queryTaskResponse.results.documents.values().forEach(resultCompute -> {
        ComputeState cs = Utils.fromJson(resultCompute, ComputeState.class);
        assertPropertiesOfComputeState(cs, ENROLLMENT_NUMNBER, cs.customProperties.get(AzureConstants.AZURE_SUBSCRIPTION_ID_KEY), ACCOUNT_EMAIL_ID_1, this.compute.endpointLink, this.compute.tenantLinks);
        this.createdComputeLinks.add(cs.documentSelfLink);
    });
}

Example 73 with ComputeState

use of com.vmware.photon.controller.model.resources.ComputeService.ComputeState in project photon-model by vmware.

the class AzureSubscriptionsEnumerationServiceTest method testAddSameAzureSubscriptions.

private void testAddSameAzureSubscriptions() throws Throwable {
    // Request for creating computes for existing Azure Subscriptions
    AzureSubscription subscription1 = getAzureSubscription(SUBSCRIPTION_ID_1, ACCOUNT_EMAIL_ID_1);
    AzureSubscription subscription2 = getAzureSubscription(SUBSCRIPTION_ID_2, ACCOUNT_EMAIL_ID_2);
    AzureSubscription existingSubscription2 = getAzureSubscription(SUBSCRIPTION_EXISTING_2, ACCOUNT_EMAIL_ID_2);
    createAzureCostComputesForSubscriptions(Arrays.asList(subscription1, subscription2, existingSubscription2));
    // Query for Azure Computes created with CLIENT_ID as enrollment Number
    QueryTask task = createQueryTaskForAzureComputes(ENROLLMENT_NUMNBER, Collections.singletonList(TENANT_ID));
    QueryTask queryTaskResponse = executQuerySynchronously(task);
    assertQueryTaskResponse(queryTaskResponse, 6);
    // Remove compute for existing subscription
    ComputeState existingSubsCs1 = Utils.fromJson(queryTaskResponse.results.documents.remove(this.existingSubsComputeLink1), ComputeState.class);
    Assert.assertNotNull(existingSubsCs1);
    Assert.assertNull(existingSubsCs1.customProperties.get(PhotonModelConstants.AUTO_DISCOVERED_ENTITY));
    ComputeState existingSubsCs2 = Utils.fromJson(queryTaskResponse.results.documents.remove(this.existingSubsComputeLink2), ComputeState.class);
    Assert.assertNotNull(existingSubsCs2);
    Assert.assertNull(existingSubsCs1.customProperties.get(PhotonModelConstants.AUTO_DISCOVERED_ENTITY));
    // Remove the already asserted computes
    this.createdComputeLinks.stream().forEach(computeLnk -> {
        queryTaskResponse.results.documents.remove(computeLnk);
    });
    // Get and assert the returned compute, which is created newly
    ComputeState cs = Utils.fromJson(queryTaskResponse.results.documents.values().iterator().next(), ComputeState.class);
    assertPropertiesOfComputeState(cs, ENROLLMENT_NUMNBER, SUBSCRIPTION_EXISTING_2, ACCOUNT_EMAIL_ID_2, this.compute.endpointLink, this.compute.tenantLinks);
    this.createdComputeLinks.add(cs.documentSelfLink);
}

Example 74 with ComputeState

use of com.vmware.photon.controller.model.resources.ComputeService.ComputeState in project photon-model by vmware.

the class TestAzureEnumerationTask method testEnumeration.

@Test
public void testEnumeration() throws Throwable {
    this.storageDescription = createDefaultStorageAccountDescription(this.host, this.mockedStorageAccountName, computeHost, endpointState);
    this.resourceGroupState = createDefaultResourceGroupState(this.host, this.mockedStorageAccountName, computeHost, endpointState, ResourceGroupStateType.AzureResourceGroup);
    this.diskState = createDefaultDiskState(this.host, this.mockedStorageAccountName, this.mockedStorageAccountName, computeHost, endpointState);
    // create an Azure VM compute resource (this also creates a disk and a storage account)
    this.vmState = createDefaultVMResource(this.host, azureVMName, computeHost, endpointState, NIC_SPEC);
    // kick off a provision task to do the actual VM creation
    ProvisionComputeTaskState provisionTask = new ProvisionComputeTaskState();
    provisionTask.computeLink = this.vmState.documentSelfLink;
    provisionTask.isMockRequest = this.isMock;
    provisionTask.taskSubStage = SubStage.CREATING_HOST;
    ProvisionComputeTaskState outTask = TestUtils.doPost(this.host, provisionTask, ProvisionComputeTaskState.class, UriUtils.buildUri(this.host, ProvisionComputeTaskService.FACTORY_LINK));
    this.host.waitForFinishedTask(ProvisionComputeTaskState.class, outTask.documentSelfLink);
    // Check resources have been created
    // expected VM count = 2 (1 compute host instance + 1 vm compute state)
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 2, ComputeService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 1, StorageDescriptionService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 1, ResourceGroupService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 1, DiskService.FACTORY_LINK, false);
    this.numberOfVMsToDelete++;
    if (this.isMock) {
        runEnumeration();
        deleteVMs(this.host, this.vmState.documentSelfLink, this.isMock, 1);
        this.vmState = null;
        ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 1, ComputeService.FACTORY_LINK, false);
        deleteServiceDocument(this.host, this.storageDescription.documentSelfLink);
        this.storageDescription = null;
        ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 0, StorageDescriptionService.FACTORY_LINK, true);
        deleteServiceDocument(this.host, this.resourceGroupState.documentSelfLink);
        this.resourceGroupState = null;
        ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 0, ResourceGroupService.FACTORY_LINK, false);
        deleteServiceDocument(this.host, this.diskState.documentSelfLink);
        this.diskState = null;
        ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, 0, DiskService.FACTORY_LINK, false);
        return;
    }
    createStaleResource();
    tagAzureResources();
    // stale resources + 1 compute host instance + 1 vm compute state
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_VM_RESOURCES_COUNT + 2, ComputeService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_STORAGE_ACCOUNTS_COUNT + 1, StorageDescriptionService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_CONTAINERS_COUNT + STALE_RG_COUNT + 1, ResourceGroupService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_DISKS_COUNT + 1, DiskService.FACTORY_LINK, false);
    // 1 network per each stale vm resource + 1 network for original vm compute state.
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_VM_RESOURCES_COUNT + 1, NetworkService.FACTORY_LINK, false);
    // 1 subnet per network, 1 network per each stale vm resource + 1 subnet for the original
    // compute state.
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_VM_RESOURCES_COUNT + 1, SubnetService.FACTORY_LINK, false);
    // 1 network per each stale vm resource + 1 network for original vm compute state.
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_VM_RESOURCES_COUNT + 1, NetworkInterfaceService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_SECURITY_GROUPS_COUNT, SecurityGroupService.FACTORY_LINK, false);
    this.vmCount = getAzureVMCount(this.computeManagementClient);
    this.host.log(Level.INFO, "Initial VM Count: %d", this.vmCount);
    runEnumeration();
    assertRemoteResources();
    assertInternalTagResources();
    assertStaleResources();
    // VM count + 1 compute host instance
    this.vmCount = this.vmCount + 1;
    ServiceDocumentQueryResult result = ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, this.vmCount, ComputeService.FACTORY_LINK, false);
    // validate type field for enumerated VMs
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).filter(c -> !c.documentSelfLink.equals(computeHost.documentSelfLink));
    // validate internal tags for enumerated VMs
    TagService.TagState expectedInternalTypeTag = newTagState(TAG_KEY_TYPE, AzureConstants.AzureResourceType.azure_vm.toString(), EnumSet.of(SYSTEM), endpointState.tenantLinks);
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).filter(c -> c.type.equals(ComputeType.VM_GUEST)).forEach(c -> {
        assertNotNull("ComputeState tagLinks is NULL", c.tagLinks);
        assertTrue(String.format("ComputeState doesn't contain tagLink: %s", expectedInternalTypeTag.documentSelfLink), c.tagLinks.contains(expectedInternalTypeTag.documentSelfLink));
    });
    // 1 network per each stale vm resource + 1 network for original vm compute state.
    ServiceDocumentQueryResult networkResults = ProvisioningUtils.queryAllFactoryResources(this.host, NetworkService.FACTORY_LINK);
    // validate internal tags for enumerated networks
    TagService.TagState expectedNetworkInternalTypeTag = newTagState(TAG_KEY_TYPE, NETWORK_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    networkResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), NetworkState.class)).forEach(c -> {
        assertNotNull("NetworkState tagLinks is NULL", c.tagLinks);
        assertTrue(String.format("NetworkState doesn't contain tagLink: %s", expectedNetworkInternalTypeTag.documentSelfLink), c.tagLinks.contains(expectedNetworkInternalTypeTag.documentSelfLink));
    });
    // 1 subnet per network, 1 network per each stale vm resource + 1 subnet for the original
    // compute state.
    ServiceDocumentQueryResult subnetResults = ProvisioningUtils.queryAllFactoryResources(this.host, SubnetService.FACTORY_LINK);
    // validate internal tags for enumerated subnets
    TagService.TagState expectedSubnetInternalTypeTag = newTagState(TAG_KEY_TYPE, SUBNET_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    subnetResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), SubnetState.class)).forEach(c -> {
        assertNotNull("SubnetState tagLinks is NULL", c.tagLinks);
        assertTrue(String.format("SubnetState doesn't contain tagLink: %s", expectedSubnetInternalTypeTag.documentSelfLink), c.tagLinks.contains(expectedSubnetInternalTypeTag.documentSelfLink));
    });
    ServiceDocumentQueryResult nicResults = ProvisioningUtils.queryAllFactoryResources(this.host, NetworkInterfaceService.FACTORY_LINK);
    // validate internal tags for enumerated network interfaces
    TagService.TagState expectedNicInternalTypeTag = newTagState(TAG_KEY_TYPE, NETWORK_INTERFACE_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    nicResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), NetworkInterfaceState.class)).filter(c -> c.regionId != null).forEach(c -> {
        assertNotNull("NetworkInterfaceState tagLinks is NULL", c.tagLinks);
        assertTrue(String.format("NetworkInterfaceState doesn't contain tagLink: %s", expectedNicInternalTypeTag.documentSelfLink), c.tagLinks.contains(expectedNicInternalTypeTag.documentSelfLink));
    });
    // validate internal tags for disks
    ServiceDocumentQueryResult diskResults = ProvisioningUtils.queryAllFactoryResources(this.host, DiskService.FACTORY_LINK);
    TagService.TagState expectedManagedDiskInternalTypeTag = newTagState(TAG_KEY_TYPE, azure_managed_disk.toString(), false, endpointState.tenantLinks);
    diskResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), DiskState.class)).filter(d -> d.customProperties != null && d.customProperties.containsKey(AzureConstants.AZURE_MANAGED_DISK_TYPE)).forEach(c -> {
        assertNotNull("tagLinks is null for disk", c.tagLinks);
        assertTrue("disk doesn't contain correct internal tagLink", c.tagLinks.contains(expectedManagedDiskInternalTypeTag.documentSelfLink));
    });
    // validate environment name field for enumerated VMs
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).forEach(c -> assertEquals(ComputeDescription.ENVIRONMENT_NAME_AZURE, c.environmentName));
    // validate Security Group tagLinks
    ServiceDocumentQueryResult securityGroupResults = ProvisioningUtils.queryAllFactoryResources(this.host, SecurityGroupService.FACTORY_LINK);
    securityGroupResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), SecurityGroupService.SecurityGroupState.class)).forEach(c -> {
        if (c.tagLinks != null) {
            for (String tag : c.tagLinks) {
                assertTrue(tag.startsWith(TagService.FACTORY_LINK));
            }
        }
    });
    for (Entry<String, Object> key : result.documents.entrySet()) {
        ComputeState document = Utils.fromJson(key.getValue(), ComputeState.class);
        if (!document.documentSelfLink.equals(computeHost.documentSelfLink) && !document.documentSelfLink.equals(this.vmState.documentSelfLink) && document.id.toLowerCase().contains(CUSTOM_DIAGNOSTIC_ENABLED_VM.toLowerCase())) {
            this.enumeratedComputeLink = document.documentSelfLink;
            break;
        }
    }
    try {
        // Test stats for the VM that was just enumerated from Azure.
        this.host.log(Level.INFO, "Collecting stats for VM [%s]-[%s]", CUSTOM_DIAGNOSTIC_ENABLED_VM, this.enumeratedComputeLink);
        this.host.setTimeoutSeconds(300);
        if (this.enumeratedComputeLink != null) {
            this.host.waitFor("Error waiting for VM stats", () -> {
                try {
                    issueStatsRequest(this.enumeratedComputeLink, false);
                } catch (Throwable t) {
                    return false;
                }
                return true;
            });
        }
        // Test stats for the compute host.
        this.host.log(Level.INFO, "Collecting stats for host [%s]", computeHost.documentSelfLink);
        this.host.waitFor("Error waiting for host stats", () -> {
            try {
                issueStatsRequest(computeHost.documentSelfLink, true);
            } catch (Throwable t) {
                return false;
            }
            return true;
        });
    } catch (Throwable te) {
        this.host.log(Level.SEVERE, te.getMessage());
    }
    // delete vm directly on azure
    this.computeManagementClient.virtualMachines().beginDelete(azureVMName, azureVMName);
    runEnumeration();
    assertResourceDisassociated(this.host, ComputeService.FACTORY_LINK, azureVMName, true);
    // clean up
    this.vmState = null;
    this.resourceManagementClient.resourceGroups().beginDelete(azureVMName);
}

Example 75 with ComputeState

use of com.vmware.photon.controller.model.resources.ComputeService.ComputeState in project photon-model by vmware.

the class TestAzureLongRunningEnumeration method testLongRunEnumeration.

@Test
public void testLongRunEnumeration() throws Throwable {
    // Log node stats at the beginning of the test
    logNodeStats(this.host.getServiceStats(this.nodeStatsUri));
    // 1. Provision VMs
    List<ProvisionComputeTaskState> taskStates = new ArrayList<>();
    for (int i = 0; i < numOfVMsToTest; i++) {
        this.storageDescriptions.add(createDefaultStorageAccountDescription(this.host, this.mockedStorageAccountName, computeHost, endpointState));
        this.resourceGroupStates.add(createDefaultResourceGroupState(this.host, this.mockedStorageAccountName, computeHost, endpointState, ResourceGroupStateType.AzureResourceGroup));
        this.diskStates.add(createDefaultDiskState(this.host, this.mockedStorageAccountName, this.mockedStorageAccountName, computeHost, endpointState));
        // create an Azure VM compute resource (this also creates a disk and a storage account)
        this.vmStates.add(createDefaultVMResource(this.host, azureVMNames.get(i), computeHost, endpointState, nicSpecs.get(i)));
        // kick off a provision task to do the actual VM creation
        ProvisionComputeTaskState provisionTask = new ProvisionComputeTaskState();
        provisionTask.computeLink = this.vmStates.get(i).documentSelfLink;
        provisionTask.isMockRequest = this.isMock;
        provisionTask.taskSubStage = SubStage.CREATING_HOST;
        taskStates.add(TestUtils.doPost(this.host, provisionTask, ProvisionComputeTaskState.class, UriUtils.buildUri(this.host, ProvisionComputeTaskService.FACTORY_LINK)));
    }
    for (ProvisionComputeTaskState taskState : taskStates) {
        this.host.waitForFinishedTask(ProvisionComputeTaskState.class, taskState.documentSelfLink);
    }
    this.host.log(Level.INFO, "VMs provisioned successfully.");
    // Check resources have been created
    // expected VM count = numOfVMsToTest + 1 (1 compute host instance + numOfVMsToTest vm compute state)
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest + 1, ComputeService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest, StorageDescriptionService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest, ResourceGroupService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest, DiskService.FACTORY_LINK, false);
    Map<String, ComputeState> azLrtComputeStates = getVMComputeStatesWithPrefix();
    Assert.assertEquals(numOfVMsToTest, azLrtComputeStates.size());
    if (this.isMock) {
        runEnumeration(this.host, computeHost.documentSelfLink, computeHost.resourcePoolLink, endpointState, this.isMock);
        for (int i = 0; i < numOfVMsToTest; i++) {
            deleteVMs(this.host, this.vmStates.get(i).documentSelfLink, this.isMock, 1);
            this.vmStates.set(i, null);
            ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest - i, ComputeService.FACTORY_LINK, false);
            deleteServiceDocument(this.host, this.storageDescriptions.get(i).documentSelfLink);
            this.storageDescriptions.set(i, null);
            ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest - i - 1, StorageDescriptionService.FACTORY_LINK, true);
            deleteServiceDocument(this.host, this.resourceGroupStates.get(i).documentSelfLink);
            this.resourceGroupStates.set(i, null);
            ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest - i - 1, ResourceGroupService.FACTORY_LINK, false);
            deleteServiceDocument(this.host, this.diskStates.get(i).documentSelfLink);
            this.diskStates.set(i, null);
            ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, numOfVMsToTest - i - 1, DiskService.FACTORY_LINK, false);
        }
        return;
    }
    // 2. Create extra resources
    createStaleResource();
    tagAzureResources();
    // stale resources + 1 compute host instance + 1 vm compute state
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_VM_RESOURCES_COUNT + (numOfVMsToTest + 1), ComputeService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_STORAGE_ACCOUNTS_COUNT + numOfVMsToTest, StorageDescriptionService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_CONTAINERS_COUNT + STALE_RG_COUNT + 1, ResourceGroupService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_DISKS_COUNT + numOfVMsToTest, DiskService.FACTORY_LINK, false);
    ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, STALE_SECURITY_GROUPS_COUNT, SecurityGroupService.FACTORY_LINK, false);
    this.vmCount = getAzureVMCount(this.computeManagementClient);
    this.host.log(Level.INFO, "Initial VM Count: %d", this.vmCount);
    // 3. Run multiple enumerations over a period of time.
    this.host.log(Level.INFO, "Waiting for multiple enumeration runs...");
    ScheduledFuture<?> enums = runEnumerationAndLogNodeStatsPeriodically();
    this.host.waitFor("Timeout while waiting for test run duration", () -> {
        TimeUnit.MINUTES.sleep(this.testRunDurationInMinutes);
        enums.cancel(false);
        return true;
    });
    this.host.waitFor("Timeout while waiting for last enumeration to clear out.", () -> {
        TimeUnit.MINUTES.sleep(1);
        return true;
    });
    verifyResourceDuplicates();
    // 4. Validate extra resources
    assertRemoteResources();
    assertStaleResources();
    ServiceDocumentQueryResult result = ProvisioningUtils.queryDocumentsAndAssertExpectedCount(this.host, this.vmCount, ComputeService.FACTORY_LINK, false);
    // validate type field for enumerated VMs
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).filter(c -> !c.documentSelfLink.equals(computeHost.documentSelfLink)).forEach(c -> assertEquals(ComputeType.VM_GUEST, c.type));
    // validate internal tags for enumerated VMs
    TagService.TagState expectedInternalTypeTag = newTagState(TAG_KEY_TYPE, AzureConstants.AzureResourceType.azure_vm.toString(), false, endpointState.tenantLinks);
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).filter(c -> c.type.equals(ComputeType.VM_GUEST)).forEach(c -> {
        assertNotNull(c.tagLinks);
        assertTrue(c.tagLinks.contains(expectedInternalTypeTag.documentSelfLink));
    });
    // validate environment name field for enumerated VMs
    result.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), ComputeState.class)).forEach(c -> assertEquals(ComputeDescription.ENVIRONMENT_NAME_AZURE, c.environmentName));
    // 5. Validate enumerated compute states have not changed.
    Map<String, ComputeState> azLrtComputeStatesEnd = getVMComputeStatesWithPrefix();
    assertTrue(numOfVMsToTest <= azLrtComputeStatesEnd.size());
    assertComputeStatesEqual(azLrtComputeStates, azLrtComputeStatesEnd);
    // 1 network per each stale vm resource + 1 network for original vm compute state.
    ServiceDocumentQueryResult networkResults = ProvisioningUtils.queryAllFactoryResources(this.host, NetworkService.FACTORY_LINK);
    // validate internal tags for enumerated networks
    TagService.TagState expectedNetworkInternalTypeTag = newTagState(TAG_KEY_TYPE, NETWORK_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    networkResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), NetworkState.class)).forEach(c -> {
        assertNotNull(c.tagLinks);
        assertTrue(c.tagLinks.contains(expectedNetworkInternalTypeTag.documentSelfLink));
    });
    // 1 subnet per network, 1 network per each stale vm resource + 1 subnet for the original
    // compute state.
    ServiceDocumentQueryResult subnetResults = ProvisioningUtils.queryAllFactoryResources(this.host, SubnetService.FACTORY_LINK);
    // validate internal tags for enumerated subnets
    TagService.TagState expectedSubnetInternalTypeTag = newTagState(TAG_KEY_TYPE, SUBNET_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    subnetResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), SubnetState.class)).forEach(c -> {
        assertNotNull(c.tagLinks);
        assertTrue(c.tagLinks.contains(expectedSubnetInternalTypeTag.documentSelfLink));
    });
    ServiceDocumentQueryResult nicResults = ProvisioningUtils.queryAllFactoryResources(this.host, NetworkInterfaceService.FACTORY_LINK);
    // validate internal tags for enumerated network interfaces
    TagService.TagState expectedNicInternalTypeTag = newTagState(TAG_KEY_TYPE, NETWORK_INTERFACE_TAG_TYPE_VALUE, false, endpointState.tenantLinks);
    nicResults.documents.entrySet().stream().map(e -> Utils.fromJson(e.getValue(), NetworkInterfaceState.class)).forEach(c -> {
        assertNotNull(c.tagLinks);
        assertTrue(c.tagLinks.contains(expectedNicInternalTypeTag.documentSelfLink));
    });
    for (int i = 0; i < numOfVMsToTest; i++) {
        this.host.log(Level.INFO, "Deleting vm: %s", azureVMNames.get(i));
        this.computeManagementClient.virtualMachines().beginDelete(azureVMNames.get(i), azureVMNames.get(i));
    }
    runEnumeration(this.host, computeHost.documentSelfLink, computeHost.resourcePoolLink, endpointState, this.isMock);
    for (int i = 0; i < numOfVMsToTest; i++) {
        assertResourceDisassociated(this.host, ComputeService.FACTORY_LINK, azureVMNames.get(i), true);
        // clean up
        this.vmStates.set(i, null);
        this.resourceManagementClient.resourceGroups().beginDelete(azureVMNames.get(i));
        this.host.log(Level.INFO, "Deleting vm resource group %s", azureVMNames.get(i));
    }
    // Log node stats at the end of the test
    logNodeStats(this.host.getServiceStats(this.nodeStatsUri));
}