Examples with OffHeapMemoryMonitor org.apache.geode.internal.cache.control.OffHeapMemoryMonitor used on opensource projects

Example 1 with OffHeapMemoryMonitor

use of org.apache.geode.internal.cache.control.OffHeapMemoryMonitor in project geode by apache.

the class MemoryThresholdsOffHeapDUnitTest method testLRLoadRejection.

/**
   * Test that LocalRegion cache Loads are not stored in the Region if the VM is in a critical
   * state, then test that they are allowed once the VM is no longer critical
   */
@Test
public void testLRLoadRejection() throws Exception {
    final Host host = Host.getHost(0);
    final VM vm = host.getVM(2);
    final String rName = getUniqueName();
    vm.invoke(() -> disconnectFromDS());
    vm.invoke(new CacheSerializableRunnable("test LocalRegion load passthrough when critical") {

        @Override
        public void run2() throws CacheException {
            getSystem(getOffHeapProperties());
            InternalResourceManager irm = (InternalResourceManager) getCache().getResourceManager();
            final OffHeapMemoryMonitor ohmm = irm.getOffHeapMonitor();
            irm.setCriticalOffHeapPercentage(90f);
            AttributesFactory<Integer, String> af = new AttributesFactory<Integer, String>();
            af.setScope(Scope.LOCAL);
            af.setOffHeap(true);
            final AtomicInteger numLoaderInvocations = new AtomicInteger(0);
            af.setCacheLoader(new CacheLoader<Integer, String>() {

                public String load(LoaderHelper<Integer, String> helper) throws CacheLoaderException {
                    numLoaderInvocations.incrementAndGet();
                    return helper.getKey().toString();
                }

                public void close() {
                }
            });
            final LocalRegion r = (LocalRegion) getCache().createRegion(rName, af.create());
            assertFalse(ohmm.getState().isCritical());
            int expectedInvocations = 0;
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            {
                Integer k = new Integer(1);
                assertEquals(k.toString(), r.get(k));
            }
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            expectedInvocations++;
            expectedInvocations++;
            r.getAll(createRanges(10, 12));
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            getCache().getLoggerI18n().fine(addExpectedExString);
            r.put("oh1", new byte[838860]);
            r.put("oh3", new byte[157287]);
            getCache().getLoggerI18n().fine(removeExpectedExString);
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to set memoryThresholdReached";
                }

                public boolean done() {
                    return r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(2);
                assertEquals(k.toString(), r.get(k));
            }
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            expectedInvocations++;
            expectedInvocations++;
            r.getAll(createRanges(13, 15));
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            getCache().getLoggerI18n().fine(addExpectedBelow);
            r.destroy("oh3");
            getCache().getLoggerI18n().fine(removeExpectedBelow);
            wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to unset memoryThresholdReached";
                }

                public boolean done() {
                    return !r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(3);
                assertEquals(k.toString(), r.get(k));
            }
            assertEquals(expectedInvocations++, numLoaderInvocations.get());
            expectedInvocations++;
            expectedInvocations++;
            r.getAll(createRanges(16, 18));
            assertEquals(expectedInvocations, numLoaderInvocations.get());
            // Do extra validation that the entry doesn't exist in the local region
            for (Integer i : createRanges(2, 2, 13, 15)) {
                if (r.containsKey(i)) {
                    fail("Expected containsKey return false for key" + i);
                }
                if (r.getEntry(i) != null) {
                    fail("Expected getEntry to return null for key" + i);
                }
            }
        }
    });
}

Example 2 with OffHeapMemoryMonitor

use of org.apache.geode.internal.cache.control.OffHeapMemoryMonitor in project geode by apache.

the class PartitionedRegionOffHeapEvictionDUnitTest method raiseFakeNotification.

@Override
protected void raiseFakeNotification() {
    ((GemFireCacheImpl) getCache()).getOffHeapEvictor().testAbortAfterLoopCount = 1;
    setEvictionPercentage(85);
    OffHeapMemoryMonitor ohmm = ((GemFireCacheImpl) getCache()).getInternalResourceManager().getOffHeapMonitor();
    ohmm.stopMonitoring(true);
    ohmm.updateStateAndSendEvent(94371840);
}

Example 3 with OffHeapMemoryMonitor

use of org.apache.geode.internal.cache.control.OffHeapMemoryMonitor in project geode by apache.

the class MemoryThresholdsOffHeapDUnitTest method testPRLoadRejection.

/**
   * Test that a Partitioned Region loader invocation is rejected if the VM with the bucket is in a
   * critical state.
   */
// GEODE-551: waitForCriterion, memory sensitive
@Category(FlakyTest.class)
@Test
public void testPRLoadRejection() throws Exception {
    final Host host = Host.getHost(0);
    final VM accessor = host.getVM(1);
    final VM ds1 = host.getVM(2);
    final String rName = getUniqueName();
    // Make sure the desired VMs will have a fresh DS. TODO: convert these from AsyncInvocation to
    // invoke
    AsyncInvocation d0 = accessor.invokeAsync(() -> disconnectFromDS());
    AsyncInvocation d1 = ds1.invokeAsync(() -> disconnectFromDS());
    d0.join();
    assertFalse(d0.exceptionOccurred());
    d1.join();
    assertFalse(d1.exceptionOccurred());
    CacheSerializableRunnable establishConnectivity = new CacheSerializableRunnable("establishcConnectivity") {

        @Override
        public void run2() throws CacheException {
            getSystem();
        }
    };
    ds1.invoke(establishConnectivity);
    accessor.invoke(establishConnectivity);
    ds1.invoke(createPR(rName, false));
    accessor.invoke(createPR(rName, true));
    final AtomicInteger expectedInvocations = new AtomicInteger(0);
    Integer ex = (Integer) accessor.invoke(new SerializableCallable("Invoke loader from accessor, non-critical") {

        public Object call() throws Exception {
            Region<Integer, String> r = getCache().getRegion(rName);
            Integer k = new Integer(1);
            Integer expectedInvocations0 = new Integer(expectedInvocations.getAndIncrement());
            // should load for new key
            assertEquals(k.toString(), r.get(k, expectedInvocations0));
            assertTrue(r.containsKey(k));
            Integer expectedInvocations1 = new Integer(expectedInvocations.get());
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations1));
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations1));
            return expectedInvocations1;
        }
    });
    expectedInvocations.set(ex.intValue());
    ex = (Integer) ds1.invoke(new SerializableCallable("Invoke loader from datastore, non-critical") {

        public Object call() throws Exception {
            Region<Integer, String> r = getCache().getRegion(rName);
            Integer k = new Integer(2);
            Integer expectedInvocations1 = new Integer(expectedInvocations.getAndIncrement());
            // should load for new key
            assertEquals(k.toString(), r.get(k, expectedInvocations1));
            assertTrue(r.containsKey(k));
            Integer expectedInvocations2 = new Integer(expectedInvocations.get());
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations2));
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations2));
            String oldVal = r.remove(k);
            assertFalse(r.containsKey(k));
            assertEquals(k.toString(), oldVal);
            return expectedInvocations2;
        }
    });
    expectedInvocations.set(ex.intValue());
    accessor.invoke(addExpectedException);
    ds1.invoke(addExpectedException);
    ex = (Integer) ds1.invoke(new SerializableCallable("Set critical state, assert local load behavior") {

        public Object call() throws Exception {
            final OffHeapMemoryMonitor ohmm = ((InternalResourceManager) getCache().getResourceManager()).getOffHeapMonitor();
            final PartitionedRegion pr = (PartitionedRegion) getCache().getRegion(rName);
            final RegionAdvisor advisor = pr.getRegionAdvisor();
            pr.put("oh1", new byte[838860]);
            pr.put("oh3", new byte[157287]);
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "verify critical state";
                }

                public boolean done() {
                    for (final ProxyBucketRegion bucket : advisor.getProxyBucketArray()) {
                        if (bucket.isBucketSick()) {
                            return true;
                        }
                    }
                    return false;
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            // reload with same key again and again
            final Integer k = new Integer(2);
            final Integer expectedInvocations3 = new Integer(expectedInvocations.getAndIncrement());
            // load
            assertEquals(k.toString(), pr.get(k, expectedInvocations3));
            assertFalse(pr.containsKey(k));
            Integer expectedInvocations4 = new Integer(expectedInvocations.getAndIncrement());
            // load
            assertEquals(k.toString(), pr.get(k, expectedInvocations4));
            assertFalse(pr.containsKey(k));
            Integer expectedInvocations5 = new Integer(expectedInvocations.get());
            // load
            assertEquals(k.toString(), pr.get(k, expectedInvocations5));
            assertFalse(pr.containsKey(k));
            return expectedInvocations5;
        }
    });
    expectedInvocations.set(ex.intValue());
    ex = (Integer) accessor.invoke(new SerializableCallable("During critical state on datastore, assert accesor load behavior") {

        public Object call() throws Exception {
            // reload with same key again and again
            final Integer k = new Integer(2);
            Integer expectedInvocations6 = new Integer(expectedInvocations.incrementAndGet());
            Region<Integer, String> r = getCache().getRegion(rName);
            // load
            assertEquals(k.toString(), r.get(k, expectedInvocations6));
            assertFalse(r.containsKey(k));
            Integer expectedInvocations7 = new Integer(expectedInvocations.incrementAndGet());
            // load
            assertEquals(k.toString(), r.get(k, expectedInvocations7));
            assertFalse(r.containsKey(k));
            return expectedInvocations7;
        }
    });
    expectedInvocations.set(ex.intValue());
    ex = (Integer) ds1.invoke(new SerializableCallable("Set safe state on datastore, assert local load behavior") {

        public Object call() throws Exception {
            final PartitionedRegion r = (PartitionedRegion) getCache().getRegion(rName);
            r.destroy("oh3");
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "verify critical state";
                }

                public boolean done() {
                    return !r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            // same key as previously used, this time is should stick
            Integer k = new Integer(3);
            Integer expectedInvocations8 = new Integer(expectedInvocations.incrementAndGet());
            // last load for 3
            assertEquals(k.toString(), r.get(k, expectedInvocations8));
            assertTrue(r.containsKey(k));
            return expectedInvocations8;
        }
    });
    expectedInvocations.set(ex.intValue());
    accessor.invoke(new SerializableCallable("Data store in safe state, assert load behavior, accessor sets critical state, assert load behavior") {

        public Object call() throws Exception {
            final OffHeapMemoryMonitor ohmm = ((InternalResourceManager) getCache().getResourceManager()).getOffHeapMonitor();
            assertFalse(ohmm.getState().isCritical());
            Integer k = new Integer(4);
            Integer expectedInvocations9 = new Integer(expectedInvocations.incrementAndGet());
            final PartitionedRegion r = (PartitionedRegion) getCache().getRegion(rName);
            // load for 4
            assertEquals(k.toString(), r.get(k, expectedInvocations9));
            assertTrue(r.containsKey(k));
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations9));
            // Go critical in accessor
            r.put("oh3", new byte[157287]);
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "verify critical state";
                }

                public boolean done() {
                    return r.memoryThresholdReached.get();
                }
            };
            k = new Integer(5);
            Integer expectedInvocations10 = new Integer(expectedInvocations.incrementAndGet());
            // load for key 5
            assertEquals(k.toString(), r.get(k, expectedInvocations10));
            assertTrue(r.containsKey(k));
            // no load
            assertEquals(k.toString(), r.get(k, expectedInvocations10));
            // Clean up critical state
            r.destroy("oh3");
            wc = new WaitCriterion() {

                public String description() {
                    return "verify critical state";
                }

                public boolean done() {
                    return !ohmm.getState().isCritical();
                }
            };
            return expectedInvocations10;
        }
    });
    accessor.invoke(removeExpectedException);
    ds1.invoke(removeExpectedException);
}

Example 4 with OffHeapMemoryMonitor

use of org.apache.geode.internal.cache.control.OffHeapMemoryMonitor in project geode by apache.

the class MemoryThresholdsOffHeapDUnitTest method testDRLoadRejection.

/**
   * Test that DistributedRegion cacheLoade and netLoad are passed through to the calling thread if
   * the local VM is in a critical state. Once the VM has moved to a safe state then test that they
   * are allowed.
   */
// GEODE-438: test pollution, async actions, time sensitive,
@Category(FlakyTest.class)
// waitForCriterion, TODO: consider disconnect DS in setup
@Test
public void testDRLoadRejection() throws Exception {
    final Host host = Host.getHost(0);
    final VM replicate1 = host.getVM(1);
    final VM replicate2 = host.getVM(2);
    final String rName = getUniqueName();
    // Make sure the desired VMs will have a fresh DS.
    AsyncInvocation d1 = replicate1.invokeAsync(() -> disconnectFromDS());
    AsyncInvocation d2 = replicate2.invokeAsync(() -> disconnectFromDS());
    d1.join();
    assertFalse(d1.exceptionOccurred());
    d2.join();
    assertFalse(d2.exceptionOccurred());
    CacheSerializableRunnable establishConnectivity = new CacheSerializableRunnable("establishcConnectivity") {

        @SuppressWarnings("synthetic-access")
        @Override
        public void run2() throws CacheException {
            getSystem(getOffHeapProperties());
        }
    };
    replicate1.invoke(establishConnectivity);
    replicate2.invoke(establishConnectivity);
    CacheSerializableRunnable createRegion = new CacheSerializableRunnable("create DistributedRegion") {

        @Override
        public void run2() throws CacheException {
            // Assert some level of connectivity
            InternalDistributedSystem ds = getSystem(getOffHeapProperties());
            assertTrue(ds.getDistributionManager().getNormalDistributionManagerIds().size() >= 1);
            InternalResourceManager irm = (InternalResourceManager) getCache().getResourceManager();
            irm.setCriticalOffHeapPercentage(90f);
            AttributesFactory af = new AttributesFactory();
            af.setScope(Scope.DISTRIBUTED_ACK);
            af.setDataPolicy(DataPolicy.REPLICATE);
            af.setOffHeap(true);
            Region region = getCache().createRegion(rName, af.create());
        }
    };
    replicate1.invoke(createRegion);
    replicate2.invoke(createRegion);
    replicate1.invoke(addExpectedException);
    replicate2.invoke(addExpectedException);
    final Integer expected = (Integer) replicate1.invoke(new SerializableCallable("test Local DistributedRegion Load") {

        public Object call() throws Exception {
            final DistributedRegion r = (DistributedRegion) getCache().getRegion(rName);
            AttributesMutator<Integer, String> am = r.getAttributesMutator();
            am.setCacheLoader(new CacheLoader<Integer, String>() {

                final AtomicInteger numLoaderInvocations = new AtomicInteger(0);

                public String load(LoaderHelper<Integer, String> helper) throws CacheLoaderException {
                    Integer expectedInvocations = (Integer) helper.getArgument();
                    final int actualInvocations = this.numLoaderInvocations.getAndIncrement();
                    if (expectedInvocations.intValue() != actualInvocations) {
                        throw new CacheLoaderException("Expected " + expectedInvocations + " invocations, actual is " + actualInvocations);
                    }
                    return helper.getKey().toString();
                }

                public void close() {
                }
            });
            int expectedInvocations = 0;
            final OffHeapMemoryMonitor ohmm = ((InternalResourceManager) getCache().getResourceManager()).getOffHeapMonitor();
            assertFalse(ohmm.getState().isCritical());
            {
                Integer k = new Integer(1);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            r.put("oh1", new byte[838860]);
            r.put("oh3", new byte[157287]);
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to set memoryThreshold";
                }

                public boolean done() {
                    return r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(2);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            r.destroy("oh3");
            wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to unset memoryThreshold";
                }

                public boolean done() {
                    return !r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(3);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            return new Integer(expectedInvocations);
        }
    });
    final CacheSerializableRunnable validateData1 = new CacheSerializableRunnable("Validate data 1") {

        @Override
        public void run2() throws CacheException {
            Region r = getCache().getRegion(rName);
            Integer i1 = new Integer(1);
            assertTrue(r.containsKey(i1));
            assertNotNull(r.getEntry(i1));
            Integer i2 = new Integer(2);
            assertFalse(r.containsKey(i2));
            assertNull(r.getEntry(i2));
            Integer i3 = new Integer(3);
            assertTrue(r.containsKey(i3));
            assertNotNull(r.getEntry(i3));
        }
    };
    replicate1.invoke(validateData1);
    replicate2.invoke(validateData1);
    replicate2.invoke(new SerializableCallable("test DistributedRegion netLoad") {

        public Object call() throws Exception {
            final DistributedRegion r = (DistributedRegion) getCache().getRegion(rName);
            final OffHeapMemoryMonitor ohmm = ((InternalResourceManager) getCache().getResourceManager()).getOffHeapMonitor();
            assertFalse(ohmm.getState().isCritical());
            int expectedInvocations = expected.intValue();
            {
                Integer k = new Integer(4);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            // Place in a critical state for the next test
            r.put("oh3", new byte[157287]);
            WaitCriterion wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to set memoryThreshold";
                }

                public boolean done() {
                    return r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(5);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            r.destroy("oh3");
            wc = new WaitCriterion() {

                public String description() {
                    return "expected region " + r + " to unset memoryThreshold";
                }

                public boolean done() {
                    return !r.memoryThresholdReached.get();
                }
            };
            Wait.waitForCriterion(wc, 30 * 1000, 10, true);
            {
                Integer k = new Integer(6);
                assertEquals(k.toString(), r.get(k, new Integer(expectedInvocations++)));
            }
            return new Integer(expectedInvocations);
        }
    });
    replicate1.invoke(removeExpectedException);
    replicate2.invoke(removeExpectedException);
    final CacheSerializableRunnable validateData2 = new CacheSerializableRunnable("Validate data 2") {

        @Override
        public void run2() throws CacheException {
            Region<Integer, String> r = getCache().getRegion(rName);
            Integer i4 = new Integer(4);
            assertTrue(r.containsKey(i4));
            assertNotNull(r.getEntry(i4));
            Integer i5 = new Integer(5);
            assertFalse(r.containsKey(i5));
            assertNull(r.getEntry(i5));
            Integer i6 = new Integer(6);
            assertTrue(r.containsKey(i6));
            assertNotNull(r.getEntry(i6));
        }
    };
    replicate1.invoke(validateData2);
    replicate2.invoke(validateData2);
}

Example 5 with OffHeapMemoryMonitor

use of org.apache.geode.internal.cache.control.OffHeapMemoryMonitor in project geode by apache.

the class MemoryThresholdsOffHeapDUnitTest method doClientServerTest.

private void doClientServerTest(final String regionName, boolean createPR) throws Exception {
    // create region on the server
    final Host host = Host.getHost(0);
    final VM server = host.getVM(0);
    final VM client = host.getVM(1);
    final Object bigKey = -1;
    final Object smallKey = -2;
    final int port = startCacheServer(server, 0f, 90f, regionName, createPR, false, 0);
    startClient(client, server, port, regionName);
    doPuts(client, regionName, false, /* catchServerException */
    false);
    doPutAlls(client, regionName, false, /* catchServerException */
    false, /* catchLowMemoryException */
    Range.DEFAULT);
    // make the region sick in the server
    final long bytesUsedAfterSmallKey = (long) server.invoke(new SerializableCallable() {

        @Override
        public Object call() throws Exception {
            InternalResourceManager irm = ((GemFireCacheImpl) getCache()).getInternalResourceManager();
            final OffHeapMemoryMonitor ohm = irm.getOffHeapMonitor();
            assertTrue(ohm.getState().isNormal());
            getCache().getLoggerI18n().fine(addExpectedExString);
            final LocalRegion r = (LocalRegion) getRootRegion().getSubregion(regionName);
            final long bytesUsedAfterSmallKey;
            {
                OffHeapMemoryMonitorObserverImpl _testHook = new OffHeapMemoryMonitorObserverImpl();
                ohm.testHook = _testHook;
                try {
                    r.put(smallKey, "1234567890");
                    bytesUsedAfterSmallKey = _testHook.verifyBeginUpdateMemoryUsed(false);
                } finally {
                    ohm.testHook = null;
                }
            }
            {
                final OffHeapMemoryMonitorObserverImpl th = new OffHeapMemoryMonitorObserverImpl();
                ohm.testHook = th;
                try {
                    r.put(bigKey, new byte[943720]);
                    th.verifyBeginUpdateMemoryUsed(bytesUsedAfterSmallKey + 943720 + 8, true);
                    WaitCriterion waitForCritical = new WaitCriterion() {

                        public boolean done() {
                            return th.checkUpdateStateAndSendEventBeforeProcess(bytesUsedAfterSmallKey + 943720 + 8, MemoryState.EVICTION_DISABLED_CRITICAL);
                        }

                        @Override
                        public String description() {
                            return null;
                        }
                    };
                    Wait.waitForCriterion(waitForCritical, 30 * 1000, 9, false);
                    th.validateUpdateStateAndSendEventBeforeProcess(bytesUsedAfterSmallKey + 943720 + 8, MemoryState.EVICTION_DISABLED_CRITICAL);
                } finally {
                    ohm.testHook = null;
                }
            }
            WaitCriterion wc;
            if (r instanceof PartitionedRegion) {
                final PartitionedRegion pr = (PartitionedRegion) r;
                final int bucketId = PartitionedRegionHelper.getHashKey(pr, null, bigKey, null, null);
                wc = new WaitCriterion() {

                    @Override
                    public String description() {
                        return "Expected to go critical: isCritical=" + ohm.getState().isCritical();
                    }

                    @Override
                    public boolean done() {
                        if (!ohm.getState().isCritical())
                            return false;
                        // Only done once the bucket has been marked sick
                        try {
                            pr.getRegionAdvisor().checkIfBucketSick(bucketId, bigKey);
                            return false;
                        } catch (LowMemoryException ignore) {
                            return true;
                        }
                    }
                };
            } else {
                wc = new WaitCriterion() {

                    @Override
                    public String description() {
                        return "Expected to go critical: isCritical=" + ohm.getState().isCritical() + " memoryThresholdReached=" + r.memoryThresholdReached.get();
                    }

                    @Override
                    public boolean done() {
                        return ohm.getState().isCritical() && r.memoryThresholdReached.get();
                    }
                };
            }
            Wait.waitForCriterion(wc, 30000, 9, true);
            getCache().getLoggerI18n().fine(removeExpectedExString);
            return bytesUsedAfterSmallKey;
        }
    });
    // make sure client puts are rejected
    doPuts(client, regionName, true, /* catchServerException */
    false);
    doPutAlls(client, regionName, true, /* catchServerException */
    false, /* catchLowMemoryException */
    new Range(Range.DEFAULT, Range.DEFAULT.width() + 1));
    // make the region healthy in the server
    server.invoke(new SerializableRunnable() {

        public void run() {
            InternalResourceManager irm = ((GemFireCacheImpl) getCache()).getInternalResourceManager();
            final OffHeapMemoryMonitor ohm = irm.getOffHeapMonitor();
            assertTrue(ohm.getState().isCritical());
            getCache().getLogger().fine(MemoryThresholdsOffHeapDUnitTest.this.addExpectedBelow);
            OffHeapMemoryMonitorObserverImpl _testHook = new OffHeapMemoryMonitorObserverImpl();
            ohm.testHook = _testHook;
            try {
                getRootRegion().getSubregion(regionName).destroy(bigKey);
                _testHook.verifyBeginUpdateMemoryUsed(bytesUsedAfterSmallKey, true);
            } finally {
                ohm.testHook = null;
            }
            WaitCriterion wc = new WaitCriterion() {

                @Override
                public String description() {
                    return "Expected to go normal";
                }

                @Override
                public boolean done() {
                    return ohm.getState().isNormal();
                }
            };
            Wait.waitForCriterion(wc, 30000, 9, true);
            getCache().getLogger().fine(MemoryThresholdsOffHeapDUnitTest.this.removeExpectedBelow);
            return;
        }
    });
}