Examples with PartitionRegionInfo org.apache.geode.cache.partition.PartitionRegionInfo used on opensource projects

Example 1 with PartitionRegionInfo

use of org.apache.geode.cache.partition.PartitionRegionInfo in project geode by apache.

the class ResourceManagerDUnitTest method testGetPartitionedRegionDetails.

/**
   * Creates partitioned regions in multiple vms and fully exercises the getPartitionedRegionDetails
   * API on ResourceManager.
   */
@Test
public void testGetPartitionedRegionDetails() {
    // two regions
    final String[] regionPath = new String[] { getUniqueName() + "-PR-0", getUniqueName() + "-PR-1" };
    // numBuckets config for the two regions
    final int[] numBuckets = new int[] { 100, 90 };
    // redundantCopies config for the two regions
    final int[] redundantCopies = new int[] { 1, 0 };
    // localMaxMemory config to use for three members
    final int[] localMaxMemory = new int[] { 50, 100, 0 };
    // bucketKeys to use for making three buckets in first PR
    final Integer[] bucketKeys = new Integer[] { Integer.valueOf(0), Integer.valueOf(42), Integer.valueOf(76) };
    assertEquals(0, bucketKeys[0].hashCode());
    assertEquals(42, bucketKeys[1].hashCode());
    assertEquals(76, bucketKeys[2].hashCode());
    createRegion(Host.getHost(0).getVM(0), regionPath[0], localMaxMemory[0], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(1), regionPath[0], localMaxMemory[1], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(2), regionPath[0], localMaxMemory[2], numBuckets[0], redundantCopies[0]);
    createRegion(Host.getHost(0).getVM(0), regionPath[1], localMaxMemory[0], numBuckets[1], redundantCopies[1]);
    // 2 MB in size
    final byte[] value = new byte[1024 * 1024 * 2];
    createBuckets(0, regionPath[0], bucketKeys, value);
    // identify the members and their config values
    final InternalDistributedMember[] members = new InternalDistributedMember[3];
    final long[] memberSizes = new long[members.length];
    final int[] memberBucketCounts = new int[members.length];
    final int[] memberPrimaryCounts = new int[members.length];
    fillValidationArrays(members, memberSizes, memberBucketCounts, memberPrimaryCounts, regionPath[0]);
    // test everything here
    for (int i = 0; i < localMaxMemory.length; i++) {
        final int vm = i;
        Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {

            public void run() {
                Set<PartitionRegionInfo> detailsSet = PartitionRegionHelper.getPartitionRegionInfo(getCache());
                if (vm == 0) {
                    assertEquals(2, detailsSet.size());
                } else {
                    assertEquals(1, detailsSet.size());
                }
                // iterate over each PartitionedRegionDetails
                for (Iterator<PartitionRegionInfo> prIter = detailsSet.iterator(); prIter.hasNext(); ) {
                    PartitionRegionInfo details = prIter.next();
                    // NOTE: getRegionPath() contains the Region.SEPARATOR + regionPath
                    assertTrue("Unknown regionPath=" + details.getRegionPath(), details.getRegionPath().contains(regionPath[0]) || details.getRegionPath().contains(regionPath[1]));
                    if (details.getRegionPath().contains(regionPath[0])) {
                        assertEquals(numBuckets[0], details.getConfiguredBucketCount());
                        assertEquals(0, details.getLowRedundancyBucketCount());
                        assertEquals(redundantCopies[0], details.getConfiguredRedundantCopies());
                        assertEquals(redundantCopies[0], details.getActualRedundantCopies());
                        assertNull(details.getColocatedWith());
                        Set<PartitionMemberInfo> memberDetails = details.getPartitionMemberInfo();
                        assertNotNull(memberDetails);
                        assertEquals(localMaxMemory.length - 1, memberDetails.size());
                        // iterate over each PartitionMemberDetails (datastores only)
                        for (Iterator<PartitionMemberInfo> mbrIter = memberDetails.iterator(); mbrIter.hasNext(); ) {
                            PartitionMemberInfo mbrDetails = mbrIter.next();
                            assertNotNull(mbrDetails);
                            DistributedMember mbr = mbrDetails.getDistributedMember();
                            assertNotNull(mbr);
                            int membersIdx = -1;
                            for (int idx = 0; idx < members.length; idx++) {
                                if (mbr.equals(members[idx])) {
                                    membersIdx = idx;
                                }
                            }
                            assertEquals(localMaxMemory[membersIdx] * (1024L * 1024L), mbrDetails.getConfiguredMaxMemory());
                            assertEquals(memberSizes[membersIdx], mbrDetails.getSize());
                            assertEquals(memberBucketCounts[membersIdx], mbrDetails.getBucketCount());
                            assertEquals(memberPrimaryCounts[membersIdx], mbrDetails.getPrimaryCount());
                            if (mbr.equals(getSystem().getDistributedMember())) {
                                // PartitionMemberDetails represents the local member
                                PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(details.getRegionPath());
                                assertEquals(pr.getLocalMaxMemory() * (1024L * 1024L), mbrDetails.getConfiguredMaxMemory());
                                PartitionedRegionDataStore ds = pr.getDataStore();
                                assertNotNull(ds);
                                assertEquals(getSize(ds), mbrDetails.getSize());
                                assertEquals(ds.getBucketsManaged(), mbrDetails.getBucketCount());
                                assertEquals(ds.getNumberOfPrimaryBucketsManaged(), mbrDetails.getPrimaryCount());
                            }
                        }
                    } else {
                        // found the other PR which has only one datastore and we know
                        // this system memberId is the only entry in mbrDetails
                        assertEquals(numBuckets[1], details.getConfiguredBucketCount());
                        assertEquals(0, details.getLowRedundancyBucketCount());
                        assertEquals(redundantCopies[1], details.getConfiguredRedundantCopies());
                        assertEquals(redundantCopies[1], details.getActualRedundantCopies());
                        assertNull(details.getColocatedWith());
                        Set<PartitionMemberInfo> memberDetails = details.getPartitionMemberInfo();
                        assertNotNull(memberDetails);
                        assertEquals(1, memberDetails.size());
                        PartitionMemberInfo mbrDetails = memberDetails.iterator().next();
                        assertEquals(getSystem().getDistributedMember(), mbrDetails.getDistributedMember());
                        PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(details.getRegionPath());
                        assertEquals(pr.getLocalMaxMemory() * (1024L * 1024L), mbrDetails.getConfiguredMaxMemory());
                        PartitionedRegionDataStore ds = pr.getDataStore();
                        assertNotNull(ds);
                        assertEquals(getSize(ds), mbrDetails.getSize());
                        assertEquals(ds.getBucketsManaged(), mbrDetails.getBucketCount());
                        assertEquals(ds.getNumberOfPrimaryBucketsManaged(), mbrDetails.getPrimaryCount());
                    }
                }
            }
        });
    }
    destroyRegions(0, regionPath);
}

Example 2 with PartitionRegionInfo

use of org.apache.geode.cache.partition.PartitionRegionInfo in project geode by apache.

the class ResourceManagerDUnitTest method testResourceManagerBasics.

/**
   * Creates a cache in the controller and exercises all methods on the ResourceManager without
   * having any partitioned regions defined.
   */
@Test
public void testResourceManagerBasics() {
    Cache cache = getCache();
    // verify that getResourceManager works
    ResourceManager manager = cache.getResourceManager();
    assertNotNull(manager);
    // verify that getPartitionedRegionDetails returns empty set
    Set<PartitionRegionInfo> detailsSet = PartitionRegionHelper.getPartitionRegionInfo(cache);
    assertNotNull(detailsSet);
    assertEquals(Collections.emptySet(), detailsSet);
    ResourceListener listener = new ResourceListener() {

        public void onEvent(ResourceEvent event) {
        }
    };
    InternalResourceManager internalManager = (InternalResourceManager) manager;
    // verify that addResourceListener works
    internalManager.addResourceListener(listener);
    Set<ResourceListener> listeners = internalManager.getResourceListeners(ResourceType.HEAP_MEMORY);
    assertNotNull(listeners);
    assertEquals(1 + SYSTEM_LISTENERS, listeners.size());
    assertTrue(listeners.contains(listener));
    // verify that repeat adds result in only one entry of the listener
    internalManager.addResourceListener(ResourceType.HEAP_MEMORY, listener);
    listeners = internalManager.getResourceListeners(ResourceType.HEAP_MEMORY);
    assertEquals(1 + SYSTEM_LISTENERS, listeners.size());
    // verify that removeResourceListener works
    internalManager.removeResourceListener(listener);
    listeners = internalManager.getResourceListeners(ResourceType.HEAP_MEMORY);
    assertEquals(listeners.size(), SYSTEM_LISTENERS);
}

Example 3 with PartitionRegionInfo

use of org.apache.geode.cache.partition.PartitionRegionInfo in project geode by apache.

the class RebalanceOperationDUnitTest method moveBucketsWithUnrecoveredValuesRedundancy.

/**
   * Test to ensure that we wait for in progress write operations before moving a primary.
   * 
   * @throws CancellationException
   */
public void moveBucketsWithUnrecoveredValuesRedundancy(final boolean simulate) {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

        public void run() {
            System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "false");
            try {
                Cache cache = getCache();
                if (cache.findDiskStore("store") == null) {
                    cache.createDiskStoreFactory().setDiskDirs(getDiskDirs()).setMaxOplogSize(1).create("store");
                }
                AttributesFactory attr = new AttributesFactory();
                PartitionAttributesFactory paf = new PartitionAttributesFactory();
                attr.setDiskStoreName("store");
                attr.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
                paf.setRedundantCopies(0);
                paf.setRecoveryDelay(-1);
                paf.setStartupRecoveryDelay(-1);
                PartitionAttributes prAttr = paf.create();
                attr.setPartitionAttributes(prAttr);
                attr.setCacheLoader(new Bug40228Loader());
                cache.createRegion("region1", attr.create());
            } finally {
                System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true");
            }
        }
    };
    // Create the region in only 1 VM
    vm0.invoke(createPrRegion);
    // Create some buckets
    vm0.invoke(new SerializableRunnable("createSomeBuckets") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            region.put(Integer.valueOf(1), "A");
            region.put(Integer.valueOf(2), "A");
            region.put(Integer.valueOf(3), "A");
            region.put(Integer.valueOf(4), "A");
            region.put(Integer.valueOf(5), "A");
            region.put(Integer.valueOf(6), "A");
        }
    });
    final long[] bucketSizes = (long[]) vm0.invoke(new SerializableCallable("get sizes and close cache") {

        public Object call() {
            PartitionedRegion region = (PartitionedRegion) getCache().getRegion("region1");
            PartitionedRegionDataStore dataStore = region.getDataStore();
            long[] bucketSizes = new long[7];
            for (int i = 1; i <= 6; i++) {
                BucketRegion bucket = dataStore.getLocalBucketById(i);
                bucketSizes[i] = bucket.getTotalBytes();
                assertEquals(0, bucket.getNumOverflowBytesOnDisk());
                assertEquals(0, bucket.getNumOverflowOnDisk());
                assertEquals(1, bucket.getNumEntriesInVM());
            }
            getCache().close();
            return bucketSizes;
        }
    });
    // now recover the region
    vm0.invoke(createPrRegion);
    vm0.invoke(new SerializableRunnable("check sizes") {

        public void run() {
            PartitionedRegion region = (PartitionedRegion) getCache().getRegion("region1");
            PartitionedRegionDataStore dataStore = region.getDataStore();
            for (int i = 1; i <= 6; i++) {
                BucketRegion bucket = dataStore.getLocalBucketById(i);
                assertEquals(1, bucket.getNumOverflowOnDisk());
                assertEquals(0, bucket.getNumEntriesInVM());
                // the size recorded on disk is not the same is the in memory size, apparently
                assertTrue("Bucket size was " + bucket.getNumOverflowBytesOnDisk(), 1 < bucket.getNumOverflowBytesOnDisk());
                assertEquals(bucket.getNumOverflowBytesOnDisk(), bucket.getTotalBytes());
            }
        }
    });
    // Create the region in the other VM (should have no effect)
    vm1.invoke(createPrRegion);
    // Now simulate a rebalance
    vm0.invoke(new SerializableRunnable("simulateRebalance") {

        public void run() {
            Cache cache = getCache();
            ResourceManager manager = cache.getResourceManager();
            RebalanceResults results = doRebalance(simulate, manager);
            assertEquals(0, results.getTotalBucketCreatesCompleted());
            assertEquals(0, results.getTotalPrimaryTransfersCompleted());
            assertEquals(3, results.getTotalBucketTransfersCompleted());
            assertTrue("Transfered Bytes = " + results.getTotalBucketTransferBytes(), 0 < results.getTotalBucketTransferBytes());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(1, detailSet.size());
            PartitionRebalanceInfo details = detailSet.iterator().next();
            assertEquals(0, details.getBucketCreatesCompleted());
            assertEquals(0, details.getPrimaryTransfersCompleted());
            assertTrue(0 < details.getBucketTransferBytes());
            assertEquals(3, details.getBucketTransfersCompleted());
            Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
            assertEquals(2, afterDetails.size());
            for (PartitionMemberInfo memberDetails : afterDetails) {
                assertEquals(3, memberDetails.getBucketCount());
                assertEquals(3, memberDetails.getPrimaryCount());
            }
            if (!simulate) {
                verifyStats(manager, results);
            }
        }
    });
    if (!simulate) {
        SerializableRunnable checkRedundancyFixed = new SerializableRunnable("checkRedundancyFixed") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
                assertEquals(6, details.getCreatedBucketCount());
                assertEquals(0, details.getActualRedundantCopies());
                assertEquals(0, details.getLowRedundancyBucketCount());
                assertEquals(2, details.getPartitionMemberInfo().size());
                for (PartitionMemberInfo memberDetails : details.getPartitionMemberInfo()) {
                    assertEquals(3, memberDetails.getBucketCount());
                    assertEquals(3, memberDetails.getPrimaryCount());
                }
                // check to make sure that moving buckets didn't close the cache loader
                Bug40228Loader loader = (Bug40228Loader) cache.getRegion("region1").getAttributes().getCacheLoader();
                assertFalse(loader.isClosed());
            }
        };
        vm0.invoke(checkRedundancyFixed);
        vm1.invoke(checkRedundancyFixed);
        SerializableRunnable checkBug40228Fixed = new SerializableRunnable("checkBug40228Fixed") {

            public void run() {
                Cache cache = getCache();
                Bug40228Loader loader = (Bug40228Loader) cache.getRegion("region1").getAttributes().getCacheLoader();
                assertFalse(loader.isClosed());
                // check to make sure that closing the PR closes the cache loader
                cache.getRegion("region1").close();
                assertTrue(loader.isClosed());
            }
        };
        vm0.invoke(checkBug40228Fixed);
        vm1.invoke(checkBug40228Fixed);
    }
}

Example 4 with PartitionRegionInfo

use of org.apache.geode.cache.partition.PartitionRegionInfo in project geode by apache.

the class RebalanceOperationDUnitTest method testMoveBucketsNestedPR.

/**
   * Test to make sure we balance buckets between three hosts with redundancy
   */
@Test
public void testMoveBucketsNestedPR() {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

        public void run() {
            Region parent;
            Cache cache = getCache();
            {
                AttributesFactory attr = new AttributesFactory();
                attr.setDataPolicy(DataPolicy.REPLICATE);
                parent = cache.createRegion("parent", attr.create());
            }
            AttributesFactory attr = new AttributesFactory();
            PartitionAttributesFactory paf = new PartitionAttributesFactory();
            paf.setRedundantCopies(1);
            paf.setRecoveryDelay(-1);
            paf.setStartupRecoveryDelay(-1);
            PartitionAttributes prAttr = paf.create();
            attr.setPartitionAttributes(prAttr);
            parent.createSubregion("region1", attr.create());
        }
    };
    // Create the region in two VMs
    vm0.invoke(createPrRegion);
    vm1.invoke(createPrRegion);
    // Create some buckets
    vm0.invoke(new SerializableRunnable("createSomeBuckets") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("parent/region1");
            for (int i = 0; i < 12; i++) {
                region.put(Integer.valueOf(i), "A");
            }
        }
    });
    // Create the region in one more VM.
    vm2.invoke(createPrRegion);
    // Now simulate a rebalance
    final Long totalSize = (Long) vm0.invoke(new SerializableCallable("simulateRebalance") {

        public Object call() {
            Cache cache = getCache();
            ResourceManager manager = cache.getResourceManager();
            RebalanceResults results = doRebalance(false, manager);
            assertEquals(0, results.getTotalBucketCreatesCompleted());
            // We don't know how many primaries will move, it depends on
            // if the move bucket code moves the primary or a redundant bucket
            // assertIndexDetailsEquals(0, results.getTotalPrimaryTransfersCompleted());
            assertEquals(8, results.getTotalBucketTransfersCompleted());
            assertTrue(0 < results.getTotalBucketTransferBytes());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(1, detailSet.size());
            PartitionRebalanceInfo details = detailSet.iterator().next();
            assertEquals(0, details.getBucketCreatesCompleted());
            assertTrue(0 < details.getBucketTransferBytes());
            assertEquals(8, details.getBucketTransfersCompleted());
            long totalSize = 0;
            Set<PartitionMemberInfo> beforeDetails = details.getPartitionMemberDetailsAfter();
            for (PartitionMemberInfo memberDetails : beforeDetails) {
                totalSize += memberDetails.getSize();
            }
            long afterSize = 0;
            Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
            assertEquals(3, afterDetails.size());
            for (PartitionMemberInfo memberDetails : afterDetails) {
                assertEquals(8, memberDetails.getBucketCount());
                assertEquals(4, memberDetails.getPrimaryCount());
                afterSize += memberDetails.getSize();
            }
            assertEquals(totalSize, afterSize);
            verifyStats(manager, results);
            return Long.valueOf(totalSize);
        }
    });
    SerializableRunnable checkBalance = new SerializableRunnable("checkBalance") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("parent/region1");
            PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
            assertEquals(12, details.getCreatedBucketCount());
            assertEquals(1, details.getActualRedundantCopies());
            assertEquals(0, details.getLowRedundancyBucketCount());
            LogWriterUtils.getLogWriter().info("details=" + details.getPartitionMemberInfo());
            long afterSize = 0;
            for (PartitionMemberInfo memberDetails : details.getPartitionMemberInfo()) {
                assertEquals(8, memberDetails.getBucketCount());
                assertEquals(4, memberDetails.getPrimaryCount());
                afterSize += memberDetails.getSize();
            }
            assertEquals(totalSize.longValue(), afterSize);
        }
    };
    vm0.invoke(checkBalance);
    vm1.invoke(checkBalance);
    vm2.invoke(checkBalance);
}

Example 5 with PartitionRegionInfo

use of org.apache.geode.cache.partition.PartitionRegionInfo in project geode by apache.

the class RebalanceOperationDUnitTest method enforceZone.

/**
   * Test that we correctly use the redundancy-zone property to determine where to place redundant
   * copies of a buckets.
   * 
   * @param simulate
   */
public void enforceZone(final boolean simulate) {
    try {
        Host host = Host.getHost(0);
        VM vm0 = host.getVM(0);
        VM vm1 = host.getVM(1);
        VM vm2 = host.getVM(2);
        setRedundancyZone(vm0, "A");
        setRedundancyZone(vm1, "A");
        final DistributedMember zoneBMember = setRedundancyZone(vm2, "B");
        SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

            public void run() {
                Cache cache = getCache();
                AttributesFactory attr = new AttributesFactory();
                PartitionAttributesFactory paf = new PartitionAttributesFactory();
                paf.setRedundantCopies(1);
                paf.setRecoveryDelay(-1);
                paf.setStartupRecoveryDelay(-1);
                PartitionAttributes prAttr = paf.create();
                attr.setPartitionAttributes(prAttr);
                cache.createRegion("region1", attr.create());
            }
        };
        // Create the region in only 1 VM
        vm0.invoke(createPrRegion);
        // Create some buckets
        vm0.invoke(new SerializableRunnable("createSomeBuckets") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                region.put(Integer.valueOf(1), "A");
                region.put(Integer.valueOf(2), "A");
                region.put(Integer.valueOf(3), "A");
                region.put(Integer.valueOf(4), "A");
                region.put(Integer.valueOf(5), "A");
                region.put(Integer.valueOf(6), "A");
            }
        });
        SerializableRunnable checkLowRedundancy = new SerializableRunnable("checkLowRedundancy") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
                assertEquals(6, details.getCreatedBucketCount());
                assertEquals(0, details.getActualRedundantCopies());
                assertEquals(6, details.getLowRedundancyBucketCount());
            }
        };
        // make sure we can tell that the buckets have low redundancy
        vm0.invoke(checkLowRedundancy);
        // Create the region in the other VMs (should have no effect)
        vm1.invoke(createPrRegion);
        vm2.invoke(createPrRegion);
        // Make sure we still have low redundancy
        vm0.invoke(checkLowRedundancy);
        // Now simulate a rebalance
        vm0.invoke(new SerializableRunnable("simulateRebalance") {

            public void run() {
                Cache cache = getCache();
                ResourceManager manager = cache.getResourceManager();
                RebalanceResults results = doRebalance(simulate, manager);
                // We expect to satisfy redundancy with the zone B member
                assertEquals(6, results.getTotalBucketCreatesCompleted());
                // 2 primaries will go to vm2, leaving vm0 and vm1 with 2 primaries each
                assertEquals(2, results.getTotalPrimaryTransfersCompleted());
                // We actually *will* transfer 3 buckets to the other member in zone A, because that
                // improves
                // the balance
                assertEquals(3, results.getTotalBucketTransfersCompleted());
                Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
                assertEquals(1, detailSet.size());
                PartitionRebalanceInfo details = detailSet.iterator().next();
                assertEquals(6, details.getBucketCreatesCompleted());
                assertEquals(2, details.getPrimaryTransfersCompleted());
                assertEquals(3, details.getBucketTransfersCompleted());
                Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
                for (PartitionMemberInfo info : afterDetails) {
                    if (info.getDistributedMember().equals(zoneBMember)) {
                        assertEquals(6, info.getBucketCount());
                    } else {
                        assertEquals(3, info.getBucketCount());
                    }
                    assertEquals(2, info.getPrimaryCount());
                }
                // assertIndexDetailsEquals(0, details.getBucketTransferBytes());
                if (!simulate) {
                    verifyStats(manager, results);
                }
            }
        });
        if (!simulate) {
            checkBucketCount(vm0, "region1", 3);
            checkBucketCount(vm1, "region1", 3);
            checkBucketCount(vm2, "region1", 6);
        }
    } finally {
        disconnectFromDS();
        Invoke.invokeInEveryVM(new SerializableRunnable() {

            public void run() {
                // clear the redundancy zone setting
                disconnectFromDS();
            }
        });
    }
}