Examples with ObjectUpdateIterator com.vmware.photon.controller.model.adapters.vsphere.EnumerationClient.ObjectUpdateIterator used on opensource projects

Example 1 with ObjectUpdateIterator

use of com.vmware.photon.controller.model.adapters.vsphere.EnumerationClient.ObjectUpdateIterator in project photon-model by vmware.

the class VSphereIncrementalEnumerationService method refreshResourcesOnDatacenter.

private void refreshResourcesOnDatacenter(EnumerationClient client, EnumerationProgress ctx, TaskManager mgr) {
    Set<String> sharedDatastores = new HashSet<>();
    List<StoragePolicyOverlay> storagePolicies = new ArrayList<>();
    // put results in different buckets by type
    PropertyFilterSpec spec = client.createResourcesFilterSpec();
    CollectorDetails collectorDetails = new CollectorDetails();
    EnumerationClient.ObjectUpdateIterator resourcesIterator;
    SegregatedOverlays segregatedOverlays;
    logInfo("Processing resources on datacenter: %s", ctx.getDcLink());
    try {
        ManagedObjectReference resourcesPropertyCollector = client.createPropertyCollectorWithFilter(spec);
        // remove getObjectIterator API
        resourcesIterator = new ObjectUpdateIterator(resourcesPropertyCollector, this.connection.getVimPort(), "");
        List<ObjectUpdate> updates = new ArrayList<>();
        while (resourcesIterator.hasNext()) {
            UpdateSet page = resourcesIterator.next();
            if (null != page) {
                for (PropertyFilterUpdate propertyFilterUpdate : page.getFilterSet()) {
                    updates.addAll(propertyFilterUpdate.getObjectSet());
                }
            }
        }
        segregatedOverlays = segregateObjectUpdates(ctx, updates);
    } catch (Exception e) {
        String msg = "Error processing PropertyCollector results";
        logWarning(() -> msg + ": " + e.toString());
        mgr.patchTaskToFailure(msg, e);
        return;
    }
    // This will split the folders into two lists. Default folders (vm, host, network, datastore) have parent as datacenter
    // and are not visible in vCenter. These need not be persisted. Any other folders will have another folder as a parent
    // and will be persisted with  appropriate parent.
    // partitioningBy will always return a map with two entries, one for where the predicate is true and one for
    // where it is false. Even though the entries can be empty, they will be present in the map , i.e. map size will be
    // always be two
    Map<Boolean, List<FolderOverlay>> folderMap = new HashMap<>(segregatedOverlays.folders.stream().collect(Collectors.partitioningBy(s -> s.getParent().getType().equals(VimNames.TYPE_DATACENTER))));
    // Process true Folder and root folder list
    List<FolderOverlay> trueFolders = folderMap.get(Boolean.FALSE);
    List<FolderOverlay> rootFolders = folderMap.get(Boolean.TRUE);
    ctx.expectFolderCount(trueFolders.size());
    logInfo("Processing folders on datacenter: %s", ctx.getDcLink());
    for (FolderOverlay folder : trueFolders) {
        try {
            // The parent list will be passed along. This is to achieve the below
            // Folder A is root folder and has datacenter as the parent.
            // Folder Ac has parent as A. Since 'A' is not persisted anymore, 'Ac'
            // should have the datacenter as its parent.
            VsphereFolderEnumerationHelper.processFoundFolder(this, ctx, folder, rootFolders, client);
        } catch (Exception e) {
            logWarning(() -> "Error processing folder information : " + e.toString());
        }
    }
    // Process HostSystem list to get the datastore access level whether local / shared
    try {
        for (HostSystemOverlay hs : segregatedOverlays.hosts) {
            sharedDatastores.addAll(client.getDatastoresHostMountInfo(hs));
        }
    } catch (Exception e) {
        // We can continue as we will not know whether the datastore is local or shared which
        // is ok to proceed.
        logWarning(() -> "Error processing datastore host mount information : " + e.toString());
    }
    storagePolicies = VsphereStoragePolicyEnumerationHelper.createStorageProfileOverlays(this, client);
    // process results in topological order
    ctx.expectNetworkCount(segregatedOverlays.networks.size());
    logInfo("Processing network on datacenter: %s", ctx.getDcLink());
    for (NetworkOverlay net : segregatedOverlays.networks.values()) {
        VSphereNetworkEnumerationHelper.processFoundNetwork(this, ctx, net, segregatedOverlays.networks);
    }
    ctx.expectDatastoreCount(segregatedOverlays.datastores.size());
    logInfo("Processing datastore on datacenter: %s", ctx.getDcLink());
    for (DatastoreOverlay ds : segregatedOverlays.datastores) {
        ds.setMultipleHostAccess(sharedDatastores.contains(ds.getName()));
        VsphereDatastoreEnumerationHelper.processFoundDatastore(this, ctx, ds);
    }
    // checkpoint net & storage, they are not related currently
    try {
        ctx.getDatastoreTracker().await();
        ctx.getNetworkTracker().await();
        ctx.getFolderTracker().await();
    } catch (InterruptedException e) {
        threadInterrupted(mgr, e);
        return;
    }
    // datastore processing is complete.
    if (storagePolicies.size() > 0) {
        ctx.expectStoragePolicyCount(storagePolicies.size());
        for (StoragePolicyOverlay sp : storagePolicies) {
            VsphereStoragePolicyEnumerationHelper.processFoundStoragePolicy(this, ctx, sp);
        }
        // checkpoint for storage policy
        try {
            ctx.getStoragePolicyTracker().await();
        } catch (InterruptedException e) {
            threadInterrupted(mgr, e);
            return;
        }
    }
    ctx.expectComputeResourceCount(segregatedOverlays.clusters.size());
    for (ComputeResourceOverlay cluster : segregatedOverlays.clusters) {
        ctx.track(cluster);
        cluster.markHostAsClustered(segregatedOverlays.hosts);
        VsphereComputeResourceEnumerationHelper.processFoundComputeResource(this, ctx, cluster, client);
    }
    // checkpoint compute
    try {
        ctx.getComputeResourceTracker().await();
    } catch (InterruptedException e) {
        threadInterrupted(mgr, e);
        return;
    }
    // process clustered as well as non-clustered hosts
    ctx.expectHostSystemCount(segregatedOverlays.hosts.size());
    logInfo("Processing hosts on datacenter: %s", ctx.getDcLink());
    for (HostSystemOverlay hs : segregatedOverlays.hosts) {
        ctx.track(hs);
        VSphereHostSystemEnumerationHelper.processFoundHostSystem(this, ctx, hs, client);
    }
    // exclude all root resource pools
    // no need to collect the root resource pool
    segregatedOverlays.resourcePools.removeIf(rp -> !VimNames.TYPE_RESOURCE_POOL.equals(rp.getParent().getType()));
    MoRefKeyedMap<String> computeResourceNamesByMoref = collectComputeNames(segregatedOverlays.hosts, segregatedOverlays.clusters);
    ctx.expectResourcePoolCount(segregatedOverlays.resourcePools.size());
    logInfo("Processing resource pools on datacenter: %s", ctx.getDcLink());
    for (ResourcePoolOverlay rp : segregatedOverlays.resourcePools) {
        String ownerName = computeResourceNamesByMoref.get(rp.getOwner());
        VSphereResourcePoolEnumerationHelper.processFoundResourcePool(this, ctx, rp, ownerName, client);
    }
    // checkpoint compute
    try {
        ctx.getHostSystemTracker().await();
        ctx.getResourcePoolTracker().await();
    } catch (InterruptedException e) {
        threadInterrupted(mgr, e);
        return;
    }
    logInfo("Updating server disks on datacenter: %s", ctx.getDcLink());
    // update server disks with selfLinks of HostSystem
    for (HostSystemOverlay hostSystemOverlay : segregatedOverlays.hosts) {
        updateServerDisks(ctx, hostSystemOverlay);
    }
    logInfo("Processing VMs on datacenter: %s", ctx.getDcLink());
    spec = client.createVmFilterSpec(client.getDatacenter());
    List<VmOverlay> vmOverlayList = new ArrayList<>();
    EnumerationClient.ObjectUpdateIterator vmIterator;
    try {
        ManagedObjectReference vmPropertyCollector = client.createPropertyCollectorWithFilter(spec);
        vmIterator = new ObjectUpdateIterator(vmPropertyCollector, this.connection.getVimPort(), "");
        while (vmIterator.hasNext()) {
            UpdateSet page = vmIterator.next();
            if (null != page) {
                for (PropertyFilterUpdate propertyFilterUpdate : page.getFilterSet()) {
                    ctx.resetVmTracker();
                    for (ObjectUpdate cont : propertyFilterUpdate.getObjectSet()) {
                        if (!VimUtils.isVirtualMachine(cont.getObj())) {
                            continue;
                        }
                        VmOverlay vm = new VmOverlay(cont);
                        if (vm.isTemplate()) {
                            // templates are skipped, enumerated as "images" instead
                            continue;
                        }
                        if (vm.getInstanceUuid() == null) {
                            logWarning(() -> String.format("Cannot process a VM without" + " instanceUuid: %s", VimUtils.convertMoRefToString(vm.getId())));
                        } else {
                            ctx.getVmTracker().register();
                            vmOverlayList.add(vm);
                            VSphereVirtualMachineEnumerationHelper.processFoundVm(this, ctx, vm);
                        }
                    }
                    ctx.getVmTracker().arriveAndAwaitAdvance();
                    VSphereVMSnapshotEnumerationHelper.enumerateSnapshots(this, ctx, vmOverlayList);
                }
            }
        }
    } catch (Exception e) {
        String msg = "Error processing PropertyCollector results";
        logWarning(() -> msg + ": " + e.toString());
        mgr.patchTaskToFailure(msg, e);
        return;
    }
    // Sync disks deleted in vSphere
    deleteIndependentDisksUnavailableInVSphere(ctx, client);
    try {
        ctx.getDeleteDiskTracker().await();
    } catch (InterruptedException e) {
        threadInterrupted(mgr, e);
        return;
    }
    // if enumeration action is start and this is the initial enumeration, then store the property collectors and versions.
    if (EnumerationAction.START == ctx.getRequest().enumerationAction) {
        collectorDetails.vmCollectorVersion = vmIterator.getVersion();
        collectorDetails.vmPropertyCollector = vmIterator.getPropertyCollector();
        collectorDetails.resourcesCollectorVersion = resourcesIterator.getVersion();
        collectorDetails.resourcesPropertyCollector = resourcesIterator.getPropertyCollector();
        collectorDetails.datacenter = ctx.getRegionId();
        this.collectors.add(collectorDetails);
    }
}