use of com.emc.storageos.db.client.model.VirtualPool in project coprhd-controller by CoprHD.
the class QuotaHelper method getVPoolQuota.
/**
* Get quota for provided vpool
*
* @prereq none
*
* @param tenantId
* @param vpool
*
* @brief get vpool quota
* @return quota
*/
public QuotaOfCinder getVPoolQuota(String tenantId, VirtualPool vpool, StorageOSUser user) {
_log.debug("In getVPoolQuota");
Project project = getCinderHelper().getProject(tenantId.toString(), user);
List<URI> quotas = _dbClient.queryByType(QuotaOfCinder.class, true);
for (URI quota : quotas) {
QuotaOfCinder quotaObj = _dbClient.queryObject(QuotaOfCinder.class, quota);
URI vpoolUri = quotaObj.getVpool();
if (vpoolUri == null) {
continue;
} else if ((quotaObj.getProject() != null) && (quotaObj.getProject().toString().equalsIgnoreCase(project.getId().toString()))) {
VirtualPool pool = _dbClient.queryObject(VirtualPool.class, vpoolUri);
if ((pool != null) && (pool.getLabel().equals(vpool.getLabel())) && (vpool.getLabel() != null) && (vpool.getLabel().length() > 0)) {
return quotaObj;
}
}
}
HashMap<String, String> qMap = getCompleteDefaultConfiguration(tenantId);
return createVpoolDefaultQuota(project, vpool, qMap);
}
use of com.emc.storageos.db.client.model.VirtualPool in project coprhd-controller by CoprHD.
the class QuotaService method updateQuota.
/**
* Update a quota
*
* @prereq none
*
* @param tenant_id the URN of the tenant
* @param target_tenant_id the URN of the target tenant
* for which quota is being modified
*
* @brief Update Quota
* @return Quota details of target_tenant_id
*/
@PUT
@Produces({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
@Path("/{target_tenant_id}")
@CheckPermission(roles = { Role.SYSTEM_MONITOR, Role.TENANT_ADMIN }, acls = { ACL.ANY })
public Response updateQuota(@PathParam("tenant_id") String openstack_tenant_id, @PathParam("target_tenant_id") String openstackTargetTenantId, CinderQuotaDetails quotaUpdates, @Context HttpHeaders header) {
_log.info("Updating Quota");
Project project = getCinderHelper().getProject(openstackTargetTenantId.toString(), getUserFromContext());
HashMap<String, String> defaultQuotaMap = getQuotaHelper().getCompleteDefaultConfiguration(openstackTargetTenantId);
if (project == null) {
throw APIException.badRequests.projectWithTagNonexistent(openstackTargetTenantId);
}
long maxQuota = 0L;
if (project.getQuotaEnabled()) {
maxQuota = (long) (project.getQuota().intValue());
} else {
maxQuota = Long.valueOf(defaultQuotaMap.get(CinderConstants.ResourceQuotaDefaults.GIGABYTES.getResource()));
}
// bVpoolQuotaUpdate will be set to true if the user is updating the quota of a vpool w.r.t a project
// bVpoolQuotaUpdate will be set to false if the user is updating the quota of the project
boolean bVpoolQuotaUpdate = isVpoolQuotaUpdate(quotaUpdates.quota_set);
String vpoolName = null;
VirtualPool objVpool = null;
if (bVpoolQuotaUpdate) {
vpoolName = getVpoolName(quotaUpdates.quota_set);
_log.info("Vpool for which quota is being updated is {}", vpoolName);
objVpool = getCinderHelper().getVpool(vpoolName);
if (objVpool == null) {
_log.error("vpool with the given name doesnt exist");
throw APIException.badRequests.parameterIsNotValid(vpoolName);
}
if (!_permissionsHelper.tenantHasUsageACL(URI.create(openstackTargetTenantId), objVpool)) {
_log.error("tenant {} does not have access to vpool with the given name {}", openstackTargetTenantId, vpoolName);
throw APIException.badRequests.parameterIsNotValid(vpoolName);
}
_log.info("objVpool.getLabel() is {}", objVpool.getLabel());
}
List<URI> quotas = _dbClient.queryByType(QuotaOfCinder.class, true);
boolean noEntriesInDB = true;
for (URI quota : quotas) {
QuotaOfCinder quotaObj = _dbClient.queryObject(QuotaOfCinder.class, quota);
if ((quotaObj.getProject() != null) && (quotaObj.getProject().toString().equalsIgnoreCase(project.getId().toString()))) {
_log.info("QuotaObj being updated is {}", quotaObj.toString());
URI vpoolUri = quotaObj.getVpool();
if ((!bVpoolQuotaUpdate) && (vpoolUri != null)) {
// Hence just skip the db entry as this is not our concern.
continue;
}
if ((bVpoolQuotaUpdate) && (vpoolUri != null)) {
// The user requested quota update for a vpool w.r.t a project.
// The current db entry that we looking into has vpool entry.
// Hence we should further check if the vpool value is same as the vpool for which the user wants to set quota
VirtualPool pool = _dbClient.queryObject(VirtualPool.class, vpoolUri);
if ((pool != null) && (pool.getLabel().equals(vpoolName)) && (vpoolName != null) && (vpoolName.length() > 0)) {
if (quotaUpdates.quota_set.containsKey("gigabytes_" + vpoolName))
quotaObj.setTotalQuota(new Long(quotaUpdates.quota_set.get("gigabytes_" + vpoolName)));
if (quotaUpdates.quota_set.containsKey("volumes_" + vpoolName))
quotaObj.setVolumesLimit(new Long(quotaUpdates.quota_set.get("volumes_" + vpoolName)));
if (quotaUpdates.quota_set.containsKey("snapshots_" + vpoolName))
quotaObj.setSnapshotsLimit(new Long(quotaUpdates.quota_set.get("snapshots_" + vpoolName)));
noEntriesInDB = false;
_dbClient.updateObject(quotaObj);
return getQuotaDetailFormat(header, quotaUpdates);
}
} else if (!bVpoolQuotaUpdate) {
// The current db entry is a project quota entity.(because to reach here vpoolUri should be Null.
if (quotaUpdates.quota_set.containsKey("gigabytes"))
quotaObj.setTotalQuota(new Long(quotaUpdates.quota_set.get("gigabytes")));
if (quotaUpdates.quota_set.containsKey("volumes"))
quotaObj.setVolumesLimit(new Long(quotaUpdates.quota_set.get("volumes")));
if (quotaUpdates.quota_set.containsKey("snapshots"))
quotaObj.setSnapshotsLimit(new Long(quotaUpdates.quota_set.get("snapshots")));
noEntriesInDB = false;
_dbClient.updateObject(quotaObj);
return getQuotaDetailFormat(header, quotaUpdates);
}
}
}
if (noEntriesInDB) {
_log.info("No entries in the QuotaOfCinder column family");
QuotaOfCinder objQuotaOfCinder = new QuotaOfCinder();
objQuotaOfCinder.setProject(project.getId());
if (bVpoolQuotaUpdate) {
objQuotaOfCinder.setVpool(objVpool.getId());
_log.info("Updating Quota of Vpool");
if (quotaUpdates.quota_set.containsKey("gigabytes_" + vpoolName))
objQuotaOfCinder.setTotalQuota(new Long(quotaUpdates.quota_set.get("gigabytes_" + vpoolName)));
else
objQuotaOfCinder.setTotalQuota(Long.valueOf(defaultQuotaMap.get("gigabytes_" + vpoolName)));
if (quotaUpdates.quota_set.containsKey("volumes_" + vpoolName))
objQuotaOfCinder.setVolumesLimit(new Long(quotaUpdates.quota_set.get("volumes_" + vpoolName)));
else
objQuotaOfCinder.setVolumesLimit(Long.valueOf(defaultQuotaMap.get("volumes_" + vpoolName)));
if (quotaUpdates.quota_set.containsKey("snapshots_" + vpoolName))
objQuotaOfCinder.setSnapshotsLimit(new Long(quotaUpdates.quota_set.get("snapshots_" + vpoolName)));
else
objQuotaOfCinder.setSnapshotsLimit(Long.valueOf(defaultQuotaMap.get("snapshots_" + vpoolName)));
} else {
if (quotaUpdates.quota_set.containsKey("gigabytes"))
objQuotaOfCinder.setTotalQuota(new Long(quotaUpdates.quota_set.get("gigabytes")));
else
objQuotaOfCinder.setTotalQuota(maxQuota);
if (quotaUpdates.quota_set.containsKey("volumes"))
objQuotaOfCinder.setVolumesLimit(new Long(quotaUpdates.quota_set.get("volumes")));
else
objQuotaOfCinder.setVolumesLimit(Long.valueOf(defaultQuotaMap.get("volumes")));
if (quotaUpdates.quota_set.containsKey("snapshots"))
objQuotaOfCinder.setSnapshotsLimit(new Long(quotaUpdates.quota_set.get("snapshots")));
else
objQuotaOfCinder.setSnapshotsLimit(Long.valueOf(defaultQuotaMap.get("snapshots")));
}
objQuotaOfCinder.setId(URI.create(UUID.randomUUID().toString()));
_dbClient.createObject(objQuotaOfCinder);
return getQuotaDetailFormat(header, quotaUpdates);
}
return getQuotaDetailFormat(header, quotaUpdates);
}
use of com.emc.storageos.db.client.model.VirtualPool in project coprhd-controller by CoprHD.
the class VPlexBlockServiceApiImpl method importVirtualVolume.
/**
* Import an existing volume to a VPLEX to make it a Virtual Volume.
* Outline: 1. Determine the VPLEX(s) that could be used. 2. If this is to
* become a distributed virtual volume, get a Recommendation for the pool
* for the haVirtualArray of the Virtual Volume. 3. Create a Virtual Volume
* and link it to the existing Volume. 4. If this is a distributed virtual
* volume, create a new Volume and link it to the virtual volume. 5. Format
* the parameters and call the controller.
*
* @param arrayURI -- the URI of the Storage Array holding the existing
* Volume.
* @param importVolume -- An existing Volume that has been provisioned.
* @param vpool -- The vpool requested on the vpool change request.
* @param taskId -- The taskId
* @throws InternalException
*/
public void importVirtualVolume(URI arrayURI, Volume importVolume, VirtualPool vpool, String taskId) throws InternalException {
VirtualArray neighborhood = _dbClient.queryObject(VirtualArray.class, importVolume.getVirtualArray());
Project project = _dbClient.queryObject(Project.class, importVolume.getProject());
URI nullPoolURI = NullColumnValueGetter.getNullURI();
BlockConsistencyGroup consistencyGroup = null;
if (importVolume.getConsistencyGroup() != null) {
consistencyGroup = _dbClient.queryObject(BlockConsistencyGroup.class, importVolume.getConsistencyGroup());
}
// Determine the VPLEX(s) that could be used.
Set<URI> vplexes = ConnectivityUtil.getVPlexSystemsAssociatedWithArray(_dbClient, arrayURI);
Iterator<URI> vplexIter = vplexes.iterator();
while (vplexIter.hasNext()) {
StorageSystem vplex = _dbClient.queryObject(StorageSystem.class, vplexIter.next());
StringSet vplexVarrays = vplex.getVirtualArrays();
if ((vplexVarrays == null) || (vplexVarrays.isEmpty()) || (!vplexVarrays.contains(neighborhood.getId().toString()))) {
vplexIter.remove();
}
}
if (vplexes.isEmpty()) {
throw APIException.badRequests.noVPlexSystemsAssociatedWithStorageSystem(arrayURI);
}
// If distributed virtual volume, get a recommendation.
// Then create the volume.
List<VolumeDescriptor> descriptors = new ArrayList<VolumeDescriptor>();
URI vplexURI = null;
StorageSystem vplexSystem = null;
Volume createVolume = null;
Project vplexProject;
if (vpool.getHighAvailability().equals(VirtualPool.HighAvailabilityType.vplex_distributed.name())) {
// Determine if the user requested a specific HA VirtualArray and an associated HA VirtualPool.
VirtualArray requestedHaVarray = null;
VirtualPool requestedHaVirtualPool = vpool;
try {
if (vpool.getHaVarrayVpoolMap() != null && !vpool.getHaVarrayVpoolMap().isEmpty()) {
for (String haNH : vpool.getHaVarrayVpoolMap().keySet()) {
if (haNH.equals(NullColumnValueGetter.getNullURI().toString())) {
continue;
}
requestedHaVarray = _dbClient.queryObject(VirtualArray.class, new URI(haNH));
String haVirtualPool = vpool.getHaVarrayVpoolMap().get(haNH);
if (haVirtualPool.equals(NullColumnValueGetter.getNullURI().toString())) {
continue;
}
requestedHaVirtualPool = _dbClient.queryObject(VirtualPool.class, new URI(haVirtualPool));
break;
}
}
} catch (URISyntaxException ex) {
s_logger.error("URISyntaxException", ex);
}
VirtualPoolCapabilityValuesWrapper cosCapabilities = new VirtualPoolCapabilityValuesWrapper();
cosCapabilities.put(VirtualPoolCapabilityValuesWrapper.SIZE, getVolumeCapacity(importVolume));
cosCapabilities.put(VirtualPoolCapabilityValuesWrapper.RESOURCE_COUNT, new Integer(1));
cosCapabilities.put(VirtualPoolCapabilityValuesWrapper.THIN_PROVISIONING, importVolume.getThinlyProvisioned());
// Get the recommendations and pick one.
List<Recommendation> recommendations = getBlockScheduler().scheduleStorageForImport(neighborhood, vplexes, requestedHaVarray, requestedHaVirtualPool, cosCapabilities);
if (recommendations.isEmpty()) {
throw APIException.badRequests.noStorageFoundForVolumeMigration(requestedHaVirtualPool.getLabel(), requestedHaVarray.getLabel(), importVolume.getId());
}
Recommendation recommendation = recommendations.get(0);
VPlexRecommendation vplexRecommendation = (VPlexRecommendation) recommendation;
vplexURI = vplexRecommendation.getVPlexStorageSystem();
vplexSystem = _dbClient.queryObject(StorageSystem.class, vplexURI);
vplexProject = getVplexProject(vplexSystem, _dbClient, _tenantsService);
// Prepare the created volume.
VirtualArray haVirtualArray = _dbClient.queryObject(VirtualArray.class, vplexRecommendation.getVirtualArray());
createVolume = prepareVolumeForRequest(getVolumeCapacity(importVolume), vplexProject, haVirtualArray, vpool, vplexRecommendation.getSourceStorageSystem(), vplexRecommendation.getSourceStoragePool(), importVolume.getLabel() + "-1", ResourceOperationTypeEnum.CREATE_BLOCK_VOLUME, taskId, _dbClient);
createVolume.addInternalFlags(Flag.INTERNAL_OBJECT);
_dbClient.updateObject(createVolume);
VolumeDescriptor desc = new VolumeDescriptor(VolumeDescriptor.Type.BLOCK_DATA, createVolume.getStorageController(), createVolume.getId(), createVolume.getPool(), cosCapabilities);
descriptors.add(desc);
} else {
vplexURI = vplexes.toArray(new URI[0])[0];
vplexSystem = _dbClient.queryObject(StorageSystem.class, vplexURI);
vplexProject = getVplexProject(vplexSystem, _dbClient, _tenantsService);
}
// Prepare the VPLEX Virtual volume.
Volume vplexVolume = prepareVolumeForRequest(getVolumeCapacity(importVolume), project, neighborhood, vpool, vplexURI, nullPoolURI, importVolume.getLabel(), ResourceOperationTypeEnum.CREATE_BLOCK_VOLUME, taskId, _dbClient);
vplexVolume.setAssociatedVolumes(new StringSet());
vplexVolume.getAssociatedVolumes().add(importVolume.getId().toString());
if (createVolume != null) {
vplexVolume.getAssociatedVolumes().add(createVolume.getId().toString());
}
if (consistencyGroup != null) {
// If the volume being converted to a virtual volume has a CG, make the virtual
// volume a member of the CG.
vplexVolume.setConsistencyGroup(consistencyGroup.getId());
consistencyGroup.addRequestedTypes(Arrays.asList(BlockConsistencyGroup.Types.VPLEX.name()));
_dbClient.updateObject(consistencyGroup);
}
vplexVolume.setVirtualPool(vpool.getId());
_dbClient.updateObject(vplexVolume);
// Add a descriptor for the VPLEX_VIRT_VOLUME
VolumeDescriptor desc = new VolumeDescriptor(VolumeDescriptor.Type.VPLEX_VIRT_VOLUME, vplexURI, vplexVolume.getId(), null, null);
descriptors.add(desc);
// Add a descriptor for the import volume too!
desc = new VolumeDescriptor(VolumeDescriptor.Type.VPLEX_IMPORT_VOLUME, importVolume.getStorageController(), importVolume.getId(), importVolume.getPool(), null);
descriptors.add(desc);
// Now send the command to the controller.
try {
s_logger.info("Calling VPlex controller.");
VPlexController controller = getController();
controller.importVolume(vplexURI, descriptors, vplexProject.getId(), vplexProject.getTenantOrg().getURI(), vpool.getId(), importVolume.getLabel() + SRC_BACKEND_VOL_LABEL_SUFFIX, null, Boolean.TRUE, taskId);
} catch (InternalException ex) {
s_logger.error("ControllerException on importVolume", ex);
String errMsg = String.format("ControllerException: %s", ex.getMessage());
Operation statusUpdate = new Operation(Operation.Status.error.name(), errMsg);
_dbClient.updateTaskOpStatus(Volume.class, vplexVolume.getId(), taskId, statusUpdate);
_dbClient.markForDeletion(vplexVolume);
if (createVolume != null) {
_dbClient.markForDeletion(createVolume);
}
throw ex;
}
}
use of com.emc.storageos.db.client.model.VirtualPool in project coprhd-controller by CoprHD.
the class VPlexBlockServiceApiImpl method changeVolumeVirtualPool.
/**
* {@inheritDoc}
*/
@Override
public TaskList changeVolumeVirtualPool(List<Volume> volumes, VirtualPool vpool, VirtualPoolChangeParam vpoolChangeParam, String taskId) throws InternalException {
TaskList taskList = new TaskList();
StringBuffer notSuppReasonBuff = new StringBuffer();
VirtualPool volumeVirtualPool = _dbClient.queryObject(VirtualPool.class, volumes.get(0).getVirtualPool());
if (VirtualPoolChangeAnalyzer.isSupportedPathParamsChange(volumes.get(0), volumeVirtualPool, vpool, _dbClient, notSuppReasonBuff) || VirtualPoolChangeAnalyzer.isSupportedAutoTieringPolicyAndLimitsChange(volumes.get(0), volumeVirtualPool, vpool, _dbClient, notSuppReasonBuff)) {
taskList = createTasksForVolumes(vpool, volumes, taskId);
checkCommonVpoolUpdates(volumes, vpool, taskId);
return taskList;
}
// Check if any of the volumes passed is a VPLEX volume
// in a VPLEX CG with corresponding local consistency
// group(s) for the backend volumes.
Volume changeVPoolVolume = isVPlexVolumeInCgWithLocalType(volumes);
if (changeVPoolVolume != null) {
s_logger.info("Change vpool request has volumes in VPLEX CG with backing local CGs");
// If any of the volumes is a CG and if this is a data
// migration of the volumes, then the volumes passed must
// contain all the volumes in that CG.
VirtualPool currentVPool = _dbClient.queryObject(VirtualPool.class, changeVPoolVolume.getVirtualPool());
VirtualPoolChangeOperationEnum vpoolChange = VirtualPoolChangeAnalyzer.getSupportedVPlexVolumeVirtualPoolChangeOperation(changeVPoolVolume, currentVPool, vpool, _dbClient, new StringBuffer());
if ((vpoolChange != null) && (vpoolChange == VirtualPoolChangeOperationEnum.VPLEX_DATA_MIGRATION)) {
s_logger.info("Vpool change is a data migration");
ControllerOperationValuesWrapper operationsWrapper = new ControllerOperationValuesWrapper();
operationsWrapper.put(ControllerOperationValuesWrapper.MIGRATION_SUSPEND_BEFORE_COMMIT, vpoolChangeParam.getMigrationSuspendBeforeCommit());
operationsWrapper.put(ControllerOperationValuesWrapper.MIGRATION_SUSPEND_BEFORE_DELETE_SOURCE, vpoolChangeParam.getMigrationSuspendBeforeDeleteSource());
List<Volume> volumesNotInRG = new ArrayList<Volume>();
taskList = migrateVolumesInReplicationGroup(volumes, vpool, volumesNotInRG, null, operationsWrapper);
// Migrate volumes not in Replication Group as single volumes
if (!volumesNotInRG.isEmpty()) {
// Get the migration descriptors
List<VolumeDescriptor> descriptors = new ArrayList<VolumeDescriptor>();
for (Volume volume : volumesNotInRG) {
StorageSystem vplexStorageSystem = _dbClient.queryObject(StorageSystem.class, volume.getStorageController());
descriptors.addAll(createChangeVirtualPoolDescriptors(vplexStorageSystem, volume, vpool, taskId, null, null, operationsWrapper, true));
}
// Create the tasks
taskList.getTaskList().addAll(createTasksForVolumes(vpool, volumesNotInRG, taskId).getTaskList());
// Now we get the Orchestration controller and use it to migrate all the volumes not in a RG.
orchestrateVPoolChanges(volumesNotInRG, descriptors, taskId);
}
return taskList;
}
}
// Otherwise proceed as we normally would performing
// individual vpool changes for each volume.
String nextTaskId = taskId;
for (Volume volume : volumes) {
taskList.getTaskList().addAll(changeVolumeVirtualPool(volume.getStorageController(), volume, vpool, vpoolChangeParam, nextTaskId).getTaskList());
// Create a unique task id.
nextTaskId = UUID.randomUUID().toString();
}
return taskList;
}
use of com.emc.storageos.db.client.model.VirtualPool in project coprhd-controller by CoprHD.
the class VPlexBlockServiceApiImpl method verifyTargetSystemsForCGDataMigration.
/**
* Verifies if the valid storage pools for the target vpool specify a single
* target storage system.
*
* @param currentVPool The source vpool for a vpool change
* @param newVPool The target vpool for a vpool change.
* @param srcVarrayURI The virtual array for the volumes being migrated.
*/
private void verifyTargetSystemsForCGDataMigration(List<Volume> volumes, VirtualPool newVPool, URI srcVarrayURI) {
// Determine if the vpool change requires a migration. If the
// new vpool is null, then this is a varray migration and the
// varray is changing. In either case, the valid storage pools
// specified in the target vpool that are tagged to the passed
// varray must specify the same system so that all volumes are
// placed on the same array.
URI tgtSystemURI = null;
URI tgtHASystemURI = null;
for (Volume volume : volumes) {
VirtualPool currentVPool = _dbClient.queryObject(VirtualPool.class, volume.getVirtualPool());
if ((newVPool == null) || (VirtualPoolChangeAnalyzer.vpoolChangeRequiresMigration(currentVPool, newVPool))) {
VirtualPool vPoolToCheck = (newVPool == null ? currentVPool : newVPool);
List<StoragePool> pools = VirtualPool.getValidStoragePools(vPoolToCheck, _dbClient, true);
for (StoragePool pool : pools) {
// are tagged to the virtual array for the volumes being migrated.
if (!pool.getTaggedVirtualArrays().contains(srcVarrayURI.toString())) {
continue;
}
if (tgtSystemURI == null) {
tgtSystemURI = pool.getStorageDevice();
} else if (!tgtSystemURI.equals(pool.getStorageDevice())) {
throw APIException.badRequests.targetVPoolDoesNotSpecifyUniqueSystem();
}
}
}
// The same restriction applies to the target HA virtual pool
// when the HA side is being migrated.
URI haVArrayURI = VirtualPoolChangeAnalyzer.getHaVarrayURI(currentVPool);
if (!NullColumnValueGetter.isNullURI(haVArrayURI)) {
// The HA varray is not null so must be distributed.
VirtualPool currentHAVpool = VirtualPoolChangeAnalyzer.getHaVpool(currentVPool, _dbClient);
VirtualPool newHAVpool = VirtualPoolChangeAnalyzer.getNewHaVpool(currentVPool, newVPool, _dbClient);
if (VirtualPoolChangeAnalyzer.vpoolChangeRequiresMigration(currentHAVpool, newHAVpool)) {
List<StoragePool> haPools = VirtualPool.getValidStoragePools(newHAVpool, _dbClient, true);
for (StoragePool haPool : haPools) {
// tagged to the HA virtual array for the volumes being migrated.
if (!haPool.getTaggedVirtualArrays().contains(haVArrayURI.toString())) {
continue;
}
if (tgtHASystemURI == null) {
tgtHASystemURI = haPool.getStorageDevice();
} else if (!tgtHASystemURI.equals(haPool.getStorageDevice())) {
throw APIException.badRequests.targetHAVPoolDoesNotSpecifyUniqueSystem();
}
}
}
}
}
}
Aggregations