Examples with PartitionMemberInfo org.apache.geode.cache.partition.PartitionMemberInfo used on opensource projects

Example 6 with PartitionMemberInfo

use of org.apache.geode.cache.partition.PartitionMemberInfo in project geode by apache.

the class RebalanceOperationDUnitTest method enforceZone.

/**
   * Test that we correctly use the redundancy-zone property to determine where to place redundant
   * copies of a buckets.
   * 
   * @param simulate
   */
public void enforceZone(final boolean simulate) {
    try {
        Host host = Host.getHost(0);
        VM vm0 = host.getVM(0);
        VM vm1 = host.getVM(1);
        VM vm2 = host.getVM(2);
        setRedundancyZone(vm0, "A");
        setRedundancyZone(vm1, "A");
        final DistributedMember zoneBMember = setRedundancyZone(vm2, "B");
        SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

            public void run() {
                Cache cache = getCache();
                AttributesFactory attr = new AttributesFactory();
                PartitionAttributesFactory paf = new PartitionAttributesFactory();
                paf.setRedundantCopies(1);
                paf.setRecoveryDelay(-1);
                paf.setStartupRecoveryDelay(-1);
                PartitionAttributes prAttr = paf.create();
                attr.setPartitionAttributes(prAttr);
                cache.createRegion("region1", attr.create());
            }
        };
        // Create the region in only 1 VM
        vm0.invoke(createPrRegion);
        // Create some buckets
        vm0.invoke(new SerializableRunnable("createSomeBuckets") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                region.put(Integer.valueOf(1), "A");
                region.put(Integer.valueOf(2), "A");
                region.put(Integer.valueOf(3), "A");
                region.put(Integer.valueOf(4), "A");
                region.put(Integer.valueOf(5), "A");
                region.put(Integer.valueOf(6), "A");
            }
        });
        SerializableRunnable checkLowRedundancy = new SerializableRunnable("checkLowRedundancy") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
                assertEquals(6, details.getCreatedBucketCount());
                assertEquals(0, details.getActualRedundantCopies());
                assertEquals(6, details.getLowRedundancyBucketCount());
            }
        };
        // make sure we can tell that the buckets have low redundancy
        vm0.invoke(checkLowRedundancy);
        // Create the region in the other VMs (should have no effect)
        vm1.invoke(createPrRegion);
        vm2.invoke(createPrRegion);
        // Make sure we still have low redundancy
        vm0.invoke(checkLowRedundancy);
        // Now simulate a rebalance
        vm0.invoke(new SerializableRunnable("simulateRebalance") {

            public void run() {
                Cache cache = getCache();
                ResourceManager manager = cache.getResourceManager();
                RebalanceResults results = doRebalance(simulate, manager);
                // We expect to satisfy redundancy with the zone B member
                assertEquals(6, results.getTotalBucketCreatesCompleted());
                // 2 primaries will go to vm2, leaving vm0 and vm1 with 2 primaries each
                assertEquals(2, results.getTotalPrimaryTransfersCompleted());
                // We actually *will* transfer 3 buckets to the other member in zone A, because that
                // improves
                // the balance
                assertEquals(3, results.getTotalBucketTransfersCompleted());
                Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
                assertEquals(1, detailSet.size());
                PartitionRebalanceInfo details = detailSet.iterator().next();
                assertEquals(6, details.getBucketCreatesCompleted());
                assertEquals(2, details.getPrimaryTransfersCompleted());
                assertEquals(3, details.getBucketTransfersCompleted());
                Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
                for (PartitionMemberInfo info : afterDetails) {
                    if (info.getDistributedMember().equals(zoneBMember)) {
                        assertEquals(6, info.getBucketCount());
                    } else {
                        assertEquals(3, info.getBucketCount());
                    }
                    assertEquals(2, info.getPrimaryCount());
                }
                // assertIndexDetailsEquals(0, details.getBucketTransferBytes());
                if (!simulate) {
                    verifyStats(manager, results);
                }
            }
        });
        if (!simulate) {
            checkBucketCount(vm0, "region1", 3);
            checkBucketCount(vm1, "region1", 3);
            checkBucketCount(vm2, "region1", 6);
        }
    } finally {
        disconnectFromDS();
        Invoke.invokeInEveryVM(new SerializableRunnable() {

            public void run() {
                // clear the redundancy zone setting
                disconnectFromDS();
            }
        });
    }
}

Example 7 with PartitionMemberInfo

use of org.apache.geode.cache.partition.PartitionMemberInfo in project geode by apache.

the class RebalanceOperationDUnitTest method testRecoverRedundancyBalancingIfCreateBucketFails.

@Test
public void testRecoverRedundancyBalancingIfCreateBucketFails() {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    final DistributedMember member1 = createPrRegion(vm0, "region1", 100, null);
    vm0.invoke(new SerializableRunnable("createSomeBuckets") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            for (int i = 0; i < 1; i++) {
                region.put(Integer.valueOf(i), "A");
            }
        }
    });
    SerializableRunnable checkRedundancy = new SerializableRunnable("checkRedundancy") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
            assertEquals(1, details.getCreatedBucketCount());
            assertEquals(0, details.getActualRedundantCopies());
            assertEquals(1, details.getLowRedundancyBucketCount());
        }
    };
    vm0.invoke(checkRedundancy);
    // Now create the region in 2 more VMs
    // Let localMaxMemory(VM1) > localMaxMemory(VM2)
    // so that redundant bucket will always be attempted on VM1
    final DistributedMember member2 = createPrRegion(vm1, "region1", 100, null);
    final DistributedMember member3 = createPrRegion(vm2, "region1", 90, null);
    vm0.invoke(checkRedundancy);
    // Inject mock PRHARedundancyProvider to simulate createBucketFailures
    vm0.invoke(new SerializableRunnable("injectCreateBucketFailureAndRebalance") {

        @Override
        public void run() {
            GemFireCacheImpl cache = spy(getGemfireCache());
            // set the spied cache instance
            GemFireCacheImpl origCache = GemFireCacheImpl.setInstanceForTests(cache);
            PartitionedRegion origRegion = (PartitionedRegion) cache.getRegion("region1");
            PartitionedRegion spyRegion = spy(origRegion);
            PRHARedundancyProvider redundancyProvider = spy(new PRHARedundancyProvider(spyRegion));
            // return the spied region when ever getPartitionedRegions() is invoked
            Set<PartitionedRegion> parRegions = cache.getPartitionedRegions();
            parRegions.remove(origRegion);
            parRegions.add(spyRegion);
            doReturn(parRegions).when(cache).getPartitionedRegions();
            doReturn(redundancyProvider).when(spyRegion).getRedundancyProvider();
            // simulate create bucket fails on member2 and test if it creates on member3
            doReturn(false).when(redundancyProvider).createBackupBucketOnMember(anyInt(), eq((InternalDistributedMember) member2), anyBoolean(), anyBoolean(), any(), anyBoolean());
            // Now simulate a rebalance
            // Create operationImpl and not factory as we need spied cache to be passed to operationImpl
            RegionFilter filter = new FilterByPath(null, null);
            RebalanceOperationImpl operation = new RebalanceOperationImpl(cache, false, filter);
            operation.start();
            RebalanceResults results = null;
            try {
                results = operation.getResults(MAX_WAIT, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                Assert.fail("Interrupted waiting on rebalance", e);
            } catch (TimeoutException e) {
                Assert.fail("Timeout waiting on rebalance", e);
            }
            assertEquals(1, results.getTotalBucketCreatesCompleted());
            assertEquals(0, results.getTotalPrimaryTransfersCompleted());
            assertEquals(0, results.getTotalBucketTransferBytes());
            assertEquals(0, results.getTotalBucketTransfersCompleted());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(1, detailSet.size());
            PartitionRebalanceInfo details = detailSet.iterator().next();
            assertEquals(1, details.getBucketCreatesCompleted());
            assertEquals(0, details.getPrimaryTransfersCompleted());
            assertEquals(0, details.getBucketTransferBytes());
            assertEquals(0, details.getBucketTransfersCompleted());
            Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
            assertEquals(3, afterDetails.size());
            for (PartitionMemberInfo memberDetails : afterDetails) {
                if (memberDetails.getDistributedMember().equals(member1)) {
                    assertEquals(1, memberDetails.getBucketCount());
                    assertEquals(1, memberDetails.getPrimaryCount());
                } else if (memberDetails.getDistributedMember().equals(member2)) {
                    assertEquals(0, memberDetails.getBucketCount());
                    assertEquals(0, memberDetails.getPrimaryCount());
                } else if (memberDetails.getDistributedMember().equals(member3)) {
                    assertEquals(1, memberDetails.getBucketCount());
                    assertEquals(0, memberDetails.getPrimaryCount());
                }
            }
            ResourceManagerStats stats = cache.getInternalResourceManager().getStats();
            assertEquals(0, stats.getRebalancesInProgress());
            assertEquals(1, stats.getRebalancesCompleted());
            assertEquals(0, stats.getRebalanceBucketCreatesInProgress());
            assertEquals(results.getTotalBucketCreatesCompleted(), stats.getRebalanceBucketCreatesCompleted());
            assertEquals(1, stats.getRebalanceBucketCreatesFailed());
            // set the original cache
            GemFireCacheImpl.setInstanceForTests(origCache);
        }
    });
    SerializableRunnable checkRedundancyFixed = new SerializableRunnable("checkLowRedundancy") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
            assertEquals(1, details.getCreatedBucketCount());
            assertEquals(1, details.getActualRedundantCopies());
            assertEquals(0, details.getLowRedundancyBucketCount());
        }
    };
    vm0.invoke(checkRedundancyFixed);
    vm1.invoke(checkRedundancyFixed);
    vm2.invoke(checkRedundancyFixed);
}

Example 8 with PartitionMemberInfo

use of org.apache.geode.cache.partition.PartitionMemberInfo in project geode by apache.

the class RebalanceOperationDUnitTest method testMembershipChange.

/**
   * Test that the rebalancing operation picks up on a concurrent membership change
   */
@Test
public void testMembershipChange() {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    final VM vm2 = host.getVM(2);
    final SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

        public void run() {
            Cache cache = getCache();
            AttributesFactory attr = new AttributesFactory();
            PartitionAttributesFactory paf = new PartitionAttributesFactory();
            paf.setRedundantCopies(0);
            paf.setRecoveryDelay(-1);
            paf.setStartupRecoveryDelay(-1);
            PartitionAttributes prAttr = paf.create();
            attr.setPartitionAttributes(prAttr);
            cache.createRegion("region1", attr.create());
        }
    };
    // Create the region in only 1 VM
    vm0.invoke(createPrRegion);
    // Create some buckets
    vm0.invoke(new SerializableRunnable("createSomeBuckets") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            region.put(Integer.valueOf(1), "A");
            region.put(Integer.valueOf(2), "A");
            region.put(Integer.valueOf(3), "A");
            region.put(Integer.valueOf(4), "A");
            region.put(Integer.valueOf(5), "A");
            region.put(Integer.valueOf(6), "A");
        }
    });
    // Create the region in the other VM (should have no effect)
    vm1.invoke(createPrRegion);
    // Now do a rebalance, but start another member in the middle
    vm0.invoke(new SerializableCallable("D rebalance") {

        public Object call() throws Exception {
            GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
            InternalResourceManager manager = cache.getInternalResourceManager();
            final CountDownLatch rebalancingStarted = new CountDownLatch(1);
            final CountDownLatch memberAdded = new CountDownLatch(1);
            InternalResourceManager.setResourceObserver(new ResourceObserverAdapter() {

                boolean firstBucket = true;

                @Override
                public void movingBucket(Region region, int bucketId, DistributedMember source, DistributedMember target) {
                    if (firstBucket) {
                        firstBucket = false;
                        vm2.invoke(createPrRegion);
                    }
                }
            });
            RebalanceResults results = doRebalance(false, manager);
            assertEquals(0, results.getTotalBucketCreatesCompleted());
            assertEquals(0, results.getTotalPrimaryTransfersCompleted());
            assertEquals(4, results.getTotalBucketTransfersCompleted());
            assertTrue(0 < results.getTotalBucketTransferBytes());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(1, detailSet.size());
            PartitionRebalanceInfo details = detailSet.iterator().next();
            assertEquals(0, details.getBucketCreatesCompleted());
            assertEquals(0, details.getPrimaryTransfersCompleted());
            assertTrue(0 < details.getBucketTransferBytes());
            assertEquals(4, details.getBucketTransfersCompleted());
            Set<PartitionMemberInfo> beforeDetails = details.getPartitionMemberDetailsBefore();
            // there should have only been 2 members when the rebalancing started.
            assertEquals(2, beforeDetails.size());
            // if it was done, there should now be 3 members.
            Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
            assertEquals(3, afterDetails.size());
            for (PartitionMemberInfo memberDetails : afterDetails) {
                assertEquals(2, memberDetails.getBucketCount());
                assertEquals(2, memberDetails.getPrimaryCount());
            }
            verifyStats(manager, results);
            InternalResourceManager mgr = (InternalResourceManager) manager;
            ResourceManagerStats stats = mgr.getStats();
            assertEquals(1, stats.getRebalanceMembershipChanges());
            return null;
        }
    });
}

Example 9 with PartitionMemberInfo

use of org.apache.geode.cache.partition.PartitionMemberInfo in project geode by apache.

the class RebalanceOperationDUnitTest method runTestWaitForOperation.

/**
   * Test to ensure that we wait for in progress write operations before moving a primary.
   * 
   * @throws Exception
   */
public void runTestWaitForOperation(final Operation op) throws Exception {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    SerializableRunnable createPrRegion = new SerializableRunnable("createRegion") {

        public void run() {
            Cache cache = getCache();
            AttributesFactory attr = new AttributesFactory();
            attr.setCacheLoader(new CacheLoader() {

                public Object load(LoaderHelper helper) throws CacheLoaderException {
                    return "anobject";
                }

                public void close() {
                }
            });
            PartitionAttributesFactory paf = new PartitionAttributesFactory();
            paf.setRedundantCopies(1);
            paf.setRecoveryDelay(-1);
            paf.setStartupRecoveryDelay(-1);
            paf.setLocalMaxMemory(100);
            PartitionAttributes prAttr = paf.create();
            attr.setPartitionAttributes(prAttr);
            cache.createRegion("region1", attr.create());
        }
    };
    // Create a region in this VM with a cache writer
    // and cache loader
    Cache cache = getCache();
    AttributesFactory attr = new AttributesFactory();
    attr.setCacheLoader(new CacheLoader() {

        public Object load(LoaderHelper helper) throws CacheLoaderException {
            return "anobject";
        }

        public void close() {
        }
    });
    PartitionAttributesFactory paf = new PartitionAttributesFactory();
    paf.setRedundantCopies(1);
    paf.setRecoveryDelay(-1);
    paf.setStartupRecoveryDelay(-1);
    paf.setLocalMaxMemory(100);
    PartitionAttributes prAttr = paf.create();
    attr.setPartitionAttributes(prAttr);
    final Region region = cache.createRegion("region1", attr.create());
    // create some buckets
    region.put(Integer.valueOf(1), "A");
    region.put(Integer.valueOf(2), "A");
    BlockingCacheListener cacheWriter = new BlockingCacheListener(2);
    region.getAttributesMutator().addCacheListener(cacheWriter);
    // start two threads doing operations, one on each bucket
    // the threads will block on the cache writer. The rebalance operation
    // will try to move one of these buckets, but it shouldn't
    // be able to because of the in progress operation.
    Thread thread1 = new Thread() {

        public void run() {
            op.execute(region, Integer.valueOf(1));
        }
    };
    thread1.start();
    Thread thread2 = new Thread() {

        public void run() {
            op.execute(region, Integer.valueOf(2));
        }
    };
    thread2.start();
    cacheWriter.waitForOperationsStarted();
    SerializableRunnable checkLowRedundancy = new SerializableRunnable("checkLowRedundancy") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
            assertEquals(2, details.getCreatedBucketCount());
            assertEquals(0, details.getActualRedundantCopies());
            assertEquals(2, details.getLowRedundancyBucketCount());
        }
    };
    // make sure we can tell that the buckets have low redundancy
    checkLowRedundancy.run();
    // Create the region in the other VM (should have no effect)
    vm1.invoke(createPrRegion);
    // Make sure we still have low redundancy
    checkLowRedundancy.run();
    ResourceManager manager = cache.getResourceManager();
    RebalanceOperation rebalance = manager.createRebalanceFactory().start();
    try {
        rebalance.getResults(5, TimeUnit.SECONDS);
        fail("Operation should not have completed");
    } catch (TimeoutException expected) {
    // do nothing
    }
    cacheWriter.release();
    LogWriterUtils.getLogWriter().info("starting wait for rebalance.  Will wait for " + MAX_WAIT + " seconds");
    RebalanceResults results = rebalance.getResults(MAX_WAIT, TimeUnit.SECONDS);
    assertEquals(2, results.getTotalBucketCreatesCompleted());
    assertEquals(1, results.getTotalPrimaryTransfersCompleted());
    assertEquals(0, results.getTotalBucketTransferBytes());
    assertEquals(0, results.getTotalBucketTransfersCompleted());
    Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
    assertEquals(1, detailSet.size());
    PartitionRebalanceInfo details = detailSet.iterator().next();
    assertEquals(2, details.getBucketCreatesCompleted());
    assertEquals(1, details.getPrimaryTransfersCompleted());
    assertEquals(0, details.getBucketTransferBytes());
    assertEquals(0, details.getBucketTransfersCompleted());
    Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
    assertEquals(2, afterDetails.size());
    for (PartitionMemberInfo memberDetails : afterDetails) {
        assertEquals(2, memberDetails.getBucketCount());
        assertEquals(1, memberDetails.getPrimaryCount());
        SerializableRunnable checkRedundancyFixed = new SerializableRunnable("checkRedundancyFixed") {

            public void run() {
                Cache cache = getCache();
                Region region = cache.getRegion("region1");
                PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(region);
                assertEquals(2, details.getCreatedBucketCount());
                assertEquals(1, details.getActualRedundantCopies());
                assertEquals(0, details.getLowRedundancyBucketCount());
            }
        };
        checkRedundancyFixed.run();
        vm1.invoke(checkRedundancyFixed);
    }
}

Example 10 with PartitionMemberInfo

use of org.apache.geode.cache.partition.PartitionMemberInfo in project geode by apache.

the class RebalanceOperationDUnitTest method moveBucketsColocatedRegions.

/**
   * Test to make sure that we move buckets to balance between three hosts with a redundancy of 1
   * and two colocated regions. Makes sure that the buckets stay colocated when we move them.
   * 
   * @param simulate
   */
public void moveBucketsColocatedRegions(final boolean simulate) {
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    createPrRegion(vm0, "region1", 200, null);
    createPrRegion(vm0, "region2", 200, "region1");
    createPrRegion(vm1, "region1", 200, null);
    createPrRegion(vm1, "region2", 200, "region1");
    // Create some buckets.
    vm0.invoke(new SerializableRunnable("createSomeBuckets") {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion("region1");
            Region region2 = cache.getRegion("region2");
            for (int i = 0; i < 12; i++) {
                region.put(Integer.valueOf(i), "A");
                region2.put(Integer.valueOf(i), "A");
            }
        }
    });
    // create the leader region, but not the colocated
    // region in one of the VMs.
    createPrRegion(vm2, "region1", 200, null);
    // Simulate a rebalance, and make sure we don't
    // move any buckets yet, because we don't have
    // the colocated region in the new VMs.
    vm0.invoke(new SerializableRunnable("rebalance") {

        public void run() {
            Cache cache = getCache();
            ResourceManager manager = cache.getResourceManager();
            RebalanceResults results = doRebalance(simulate, manager);
            assertEquals(0, results.getTotalBucketCreatesCompleted());
            assertEquals(0, results.getTotalPrimaryTransfersCompleted());
            assertEquals(0, results.getTotalBucketTransferBytes());
            assertEquals(0, results.getTotalBucketTransfersCompleted());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(2, detailSet.size());
            for (PartitionRebalanceInfo details : detailSet) {
                assertEquals(0, details.getBucketCreatesCompleted());
                assertEquals(0, details.getPrimaryTransfersCompleted());
                assertEquals(0, details.getBucketTransferBytes());
                assertEquals(0, details.getBucketTransfersCompleted());
            }
        }
    });
    // now create the colocated region in the last VM.
    createPrRegion(vm2, "region2", 200, "region1");
    vm0.invoke(new SerializableRunnable("rebalance") {

        public void run() {
            Cache cache = getCache();
            ResourceManager manager = cache.getResourceManager();
            RebalanceResults results = doRebalance(simulate, manager);
            assertEquals(0, results.getTotalBucketCreatesCompleted());
            assertEquals(16, results.getTotalBucketTransfersCompleted());
            assertTrue(0 < results.getTotalBucketTransferBytes());
            Set<PartitionRebalanceInfo> detailSet = results.getPartitionRebalanceDetails();
            assertEquals(2, detailSet.size());
            for (PartitionRebalanceInfo details : detailSet) {
                assertEquals(0, details.getBucketCreatesCompleted());
                assertTrue(0 < details.getBucketTransferBytes());
                assertEquals(8, details.getBucketTransfersCompleted());
                Set<PartitionMemberInfo> afterDetails = details.getPartitionMemberDetailsAfter();
                assertEquals(3, afterDetails.size());
                for (PartitionMemberInfo memberDetails : afterDetails) {
                    assertEquals(8, memberDetails.getBucketCount());
                    assertEquals(4, memberDetails.getPrimaryCount());
                }
            }
        }
    });
    if (!simulate) {
        SerializableRunnable checkRedundancyFixed = new SerializableRunnable("checkLowRedundancy") {

            public void run() {
                Cache cache = getCache();
                PartitionedRegion region1 = (PartitionedRegion) cache.getRegion("region1");
                PartitionedRegion region2 = (PartitionedRegion) cache.getRegion("region2");
                ResourceManager manager = cache.getResourceManager();
                PartitionRegionInfo details = PartitionRegionHelper.getPartitionRegionInfo(cache.getRegion("region1"));
                assertEquals(12, details.getCreatedBucketCount());
                assertEquals(1, details.getActualRedundantCopies());
                assertEquals(0, details.getLowRedundancyBucketCount());
                details = PartitionRegionHelper.getPartitionRegionInfo(cache.getRegion("region2"));
                assertEquals(12, details.getCreatedBucketCount());
                assertEquals(1, details.getActualRedundantCopies());
                assertEquals(0, details.getLowRedundancyBucketCount());
                assertEquals(region1.getLocalPrimaryBucketsListTestOnly(), region2.getLocalPrimaryBucketsListTestOnly());
                assertEquals(region1.getLocalBucketsListTestOnly(), region2.getLocalBucketsListTestOnly());
            }
        };
        vm0.invoke(checkRedundancyFixed);
        vm1.invoke(checkRedundancyFixed);
        vm2.invoke(checkRedundancyFixed);
    }
}