Examples with CacheException org.apache.geode.cache.CacheException used on opensource projects

Example 21 with CacheException

use of org.apache.geode.cache.CacheException in project geode by apache.

the class CqQueryDUnitTest method executeCQ.

/**
   * Execute/register CQ as running.
   * 
   * @param initialResults true if initialResults are requested
   * @param expectedResultsSize if >= 0, validate results against this size
   * @param expectedErr if not null, an error we expect
   */
private void executeCQ(VM vm, final String cqName, final boolean initialResults, final int expectedResultsSize, final String expectedErr) {
    vm.invoke(new CacheSerializableRunnable("Execute CQ :" + cqName) {

        private void work() throws CacheException {
            // pause(60 * 1000);
            LogWriterUtils.getLogWriter().info("### DEBUG EXECUTE CQ START ####");
            // pause(20 * 1000);
            // Get CQ Service.
            QueryService cqService = null;
            CqQuery cq1 = null;
            // try {
            cqService = getCache().getQueryService();
            // Get CqQuery object.
            try {
                cq1 = cqService.getCq(cqName);
                if (cq1 == null) {
                    LogWriterUtils.getLogWriter().info("Failed to get CqQuery object for CQ name: " + cqName);
                    fail("Failed to get CQ " + cqName);
                } else {
                    LogWriterUtils.getLogWriter().info("Obtained CQ, CQ name: " + cq1.getName());
                    assertTrue("newCq() state mismatch", cq1.getState().isStopped());
                }
            } catch (Exception ex) {
                LogWriterUtils.getLogWriter().info("CqService is :" + cqService);
                LogWriterUtils.getLogWriter().error(ex);
                AssertionError err = new AssertionError("Failed to execute  CQ " + cqName);
                err.initCause(ex);
                throw err;
            }
            if (initialResults) {
                SelectResults cqResults = null;
                try {
                    cqResults = cq1.executeWithInitialResults();
                } catch (Exception ex) {
                    LogWriterUtils.getLogWriter().info("CqService is :" + cqService);
                    ex.printStackTrace();
                    AssertionError err = new AssertionError("Failed to execute  CQ " + cqName);
                    err.initCause(ex);
                    throw err;
                }
                LogWriterUtils.getLogWriter().info("initial result size = " + cqResults.size());
                assertTrue("executeWithInitialResults() state mismatch", cq1.getState().isRunning());
                if (expectedResultsSize >= 0) {
                    assertEquals("unexpected results size", expectedResultsSize, cqResults.size());
                }
            } else {
                try {
                    cq1.execute();
                } catch (Exception ex) {
                    AssertionError err = new AssertionError("Failed to execute  CQ " + cqName);
                    err.initCause(ex);
                    if (expectedErr == null) {
                        LogWriterUtils.getLogWriter().info("CqService is :" + cqService, err);
                    }
                    throw err;
                }
                assertTrue("execute() state mismatch", cq1.getState().isRunning());
            }
        }

        public void run2() throws CacheException {
            if (expectedErr != null) {
                getCache().getLogger().info("<ExpectedException action=add>" + expectedErr + "</ExpectedException>");
            }
            try {
                work();
            } finally {
                if (expectedErr != null) {
                    getCache().getLogger().info("<ExpectedException action=remove>" + expectedErr + "</ExpectedException>");
                }
            }
        }
    });
}

Example 22 with CacheException

use of org.apache.geode.cache.CacheException in project geode by apache.

the class CqQueryDUnitTest method createServer.

public void createServer(VM server, final int thePort, final boolean eviction, final MirrorType mirrorType) {
    SerializableRunnable createServer = new CacheSerializableRunnable("Create Cache Server") {

        public void run2() throws CacheException {
            LogWriterUtils.getLogWriter().info("### Create Cache Server. ###");
            AttributesFactory factory = new AttributesFactory();
            factory.setScope(Scope.DISTRIBUTED_ACK);
            factory.setMirrorType(mirrorType);
            // setting the eviction attributes.
            if (eviction) {
                EvictionAttributes evictAttrs = EvictionAttributes.createLRUEntryAttributes(100000, EvictionAction.OVERFLOW_TO_DISK);
                factory.setEvictionAttributes(evictAttrs);
            }
            for (int i = 0; i < regions.length; i++) {
                createRegion(regions[i], factory.createRegionAttributes());
            }
            Wait.pause(2000);
            try {
                startBridgeServer(thePort, true);
            } catch (Exception ex) {
                Assert.fail("While starting CacheServer", ex);
            }
            Wait.pause(2000);
        }
    };
    server.invoke(createServer);
}

Example 23 with CacheException

use of org.apache.geode.cache.CacheException in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInLocalRegion.

/**
   * Tests partial key putAll and removeAll to 2 servers with local region
   */
@Test
public void testPartialKeyInLocalRegion() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInLocalRegion:";
    final Host host = Host.getHost(0);
    final VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    final VM client1 = host.getVM(2);
    final VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, false, 0, null);
    final int serverPort2 = createBridgeServer(server2, regionName, 0, false, 0, null);
    createClient(client1, regionName, serverHost, new int[] { serverPort1 }, -1, -1, false, true, true);
    createClient(client2, regionName, serverHost, new int[] { serverPort1 }, -1, -1, false, true, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client1 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doPutAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof RuntimeException);
                assertTrue(soe.getCause().getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                    return false;
                }
                if (c2Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                    return false;
                }
                if (s1Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                    return false;
                }
                if (s2Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    int server1Size = getRegionSize(server1, regionName);
    int server2Size = getRegionSize(server1, regionName);
    // reset cacheWriter's count to allow another 15 keys to be created
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    // p2p putAll on DR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            // create keys
            try {
                doPutAll(regionName, title + "again:", numberOfEntries);
                fail("Expect original RuntimeException caused by cacheWriter");
            } catch (RuntimeException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    server2Size = getRegionSize(server1, regionName);
    assertEquals(server1Size + 15, server2Size);
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                    return false;
                }
                if (c2Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                    return false;
                }
                if (s1Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                    return false;
                }
                if (s2Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    // now do a removeAll that is not allowed to remove everything
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // server triggers exception after destroying 5 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(5));
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doRemoveAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_RemoveAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof RuntimeException);
                assertTrue(soe.getCause().getMessage().contains("Triggered exception as planned, destroyed 5 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15 - 5) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all destroys");
                    return false;
                }
                if (c2Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all destroys");
                    return false;
                }
                if (s1Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all destroys");
                    return false;
                }
                if (s2Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all destroys");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    // reset cacheWriter's count to allow another 5 keys to be destroyed
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // server triggers exception after destroying 5 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(5));
        }
    });
    // p2p putAll on DR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            // create keys
            try {
                doRemoveAll(regionName, title + "again:", numberOfEntries);
                fail("Expect original RuntimeException caused by cacheWriter");
            } catch (RuntimeException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, destroyed 5 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15 - 5) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all destroys");
                    return false;
                }
                if (c2Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all destroys");
                    return false;
                }
                if (s1Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all destroys");
                    return false;
                }
                if (s2Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all destroys");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    server1.invoke(removeExceptionTag1(expectedExceptions));
    server2.invoke(removeExceptionTag1(expectedExceptions));
    client1.invoke(removeExceptionTag1(expectedExceptions));
    client2.invoke(removeExceptionTag1(expectedExceptions));
    // Stop server
    stopBridgeServers(getCache());
}

Example 24 with CacheException

use of org.apache.geode.cache.CacheException in project geode by apache.

the class PutAllCSDUnitTest method doTest2NormalServerCCE.

public void doTest2NormalServerCCE(boolean CCE) throws CacheException, InterruptedException {
    final String title = "doTest2NormalServerCCE=" + CCE + ":";
    final Host host = Host.getHost(0);
    VM server1 = host.getVM(0);
    VM server2 = host.getVM(1);
    VM client1 = host.getVM(2);
    VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set notifyBySubscription=false to test local-invalidates
    int serverPort1 = createServerRegion(server1, regionName, CCE);
    int serverPort2 = createServerRegion(server2, regionName, CCE);
    createBridgeClient(client1, regionName, serverHost, new int[] { serverPort1 }, -1, 59000, true);
    createBridgeClient(client2, regionName, serverHost, new int[] { serverPort2 }, -1, 59000, true);
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
        }
    });
    // test case 1: putAll and removeAll to server1
    client1.invoke(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            doPutAll(regionName, "case1-", numberOfEntries);
            assertEquals(numberOfEntries, region.size());
            for (int i = 0; i < numberOfEntries; i++) {
                TestObject obj = (TestObject) region.getEntry("case1-" + i).getValue();
                assertEquals(i, obj.getPrice());
            }
            region.put(numberOfEntries, "specialvalue");
        }
    });
    // verify bridge server 1, its data are from client
    server1.invoke(new CacheSerializableRunnable(title + "verify Bridge Server 1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().setCacheWriter(new MyWriter("case1-"));
            region.localDestroy(numberOfEntries);
            assertEquals(numberOfEntries, region.size());
            for (int i = 0; i < numberOfEntries; i++) {
                TestObject obj = (TestObject) region.getEntry("case1-" + i).getValue();
                assertEquals(i, obj.getPrice());
            }
        }
    });
    server2.invoke(new CacheSerializableRunnable(title + "verify Bridge Server 2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().setCacheWriter(new MyWriter("case1-"));
            // normal policy will not distribute create events
            assertEquals(0, region.size());
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 removeAll") {

        @Override
        public void run2() throws CacheException {
            LocalRegion region = (LocalRegion) getRootRegion().getSubregion(regionName);
            assertEquals(numberOfEntries + 1, region.size());
            // do removeAll with some keys not exist
            doRemoveAll(regionName, "case1-", numberOfEntries * 2);
            assertEquals(1, region.size());
            // do cleanup
            region.localDestroy(numberOfEntries);
        }
    });
    // verify bridge server 1, its data are from client
    server1.invoke(new CacheSerializableRunnable(title + "verify removeAll Bridge Server 1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(0, region.size());
            MyWriter mywriter = (MyWriter) region.getAttributes().getCacheWriter();
            LogWriterUtils.getLogWriter().info("server cachewriter triggered for destroy: " + mywriter.num_destroyed);
            assertEquals(numberOfEntries, mywriter.num_destroyed);
        }
    });
    // verify bridge server 2, because its data are from distribution
    server2.invoke(new CacheSerializableRunnable(title + "verify removeAll Bridge Server 2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(0, region.size());
            MyWriter mywriter = (MyWriter) region.getAttributes().getCacheWriter();
            LogWriterUtils.getLogWriter().info("server cachewriter triggered for destroy: " + mywriter.num_destroyed);
            // beforeDestroys are only triggered at server1 since the removeAll is submitted from
            // client1
            assertEquals(0, mywriter.num_destroyed);
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client2 verify removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            checkRegionSize(region, 0);
        }
    });
    // test case 2: putAll to server1, removeAll to server2
    client1.invoke(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            doPutAll(regionName, "case2-", numberOfEntries);
            assertEquals(numberOfEntries, region.size());
            for (int i = 0; i < numberOfEntries; i++) {
                TestObject obj = (TestObject) region.getEntry("case2-" + i).getValue();
                assertEquals(i, obj.getPrice());
            }
        }
    });
    // verify bridge server 1, its data are from client
    server1.invoke(new CacheSerializableRunnable(title + "verify Bridge Server 1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(numberOfEntries, region.size());
            for (int i = 0; i < numberOfEntries; i++) {
                TestObject obj = (TestObject) region.getEntry("case2-" + i).getValue();
                assertEquals(i, obj.getPrice());
            }
        }
    });
    server2.invoke(new CacheSerializableRunnable(title + "verify Bridge Server 2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // normal policy will not distribute create events
            assertEquals(0, region.size());
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client1 removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            doRemoveAll(regionName, "case2-", numberOfEntries);
            assertEquals(0, region.size());
        }
    });
    server1.invoke(new CacheSerializableRunnable(title + "verify removeAll Bridge Server 1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(100, region.size());
        }
    });
    server2.invoke(new CacheSerializableRunnable(title + "verify removeAll Bridge Server 2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(0, region.size());
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 verify removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            checkRegionSize(region, 100);
            // do cleanup
            doRemoveAll(regionName, "case2-", numberOfEntries);
        }
    });
    // test case 3: removeAll a list with duplicated keys
    client1.invoke(new CacheSerializableRunnable(title + "put 3 keys then removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.put("case3-1", "case3-1");
            region.put("case3-2", "case3-2");
            region.put("case3-3", "case3-3");
            assertEquals(3, region.size());
            ArrayList keys = new ArrayList();
            keys.add("case3-1");
            keys.add("case3-2");
            keys.add("case3-3");
            keys.add("case3-1");
            region.removeAll(keys, "removeAllCallback");
            assertEquals(0, region.size());
        }
    });
    // verify bridge server 1, its data are from client
    server1.invoke(new CacheSerializableRunnable(title + "verify Bridge Server 1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(0, region.size());
        }
    });
    // clean up
    // Stop server
    stopBridgeServers(getCache());
}

Example 25 with CacheException

use of org.apache.geode.cache.CacheException in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInPR.

/**
   * Tests partial key putAll to 2 PR servers, because putting data at server side is different
   * between PR and LR. PR does it in postPutAll. It's not running in singleHop putAll
   */
@Test
public void testPartialKeyInPR() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInPR:";
    final Host host = Host.getHost(0);
    VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    VM client1 = host.getVM(2);
    VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, true, 0, "ds1");
    final int serverPort2 = createBridgeServer(server2, regionName, 0, true, 0, "ds1");
    createClient(client1, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    createClient(client2, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    final SharedCounter sc_server2 = new SharedCounter("server2");
    server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(server2, true, sc_server2, 10));
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client2 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    AsyncInvocation async1 = client1.invokeAsync(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            // create keys
            try {
                doPutAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                if (!(soe.getCause() instanceof PartitionOfflineException)) {
                    throw soe;
                }
                if (!soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath()))) {
                    throw soe;
                }
            }
        }
    });
    // server2 will closeCache after created 10 keys
    ThreadUtils.join(async1, 30 * 1000);
    if (async1.exceptionOccurred()) {
        Assert.fail("Aync1 get exceptions:", async1.getException());
    }
    int client1Size = getRegionSize(client1, regionName);
    // client2Size maybe more than client1Size
    int client2Size = getRegionSize(client2, regionName);
    int server1Size = getRegionSize(server1, regionName);
    LogWriterUtils.getLogWriter().info("region sizes: " + client1Size + "," + client2Size + "," + server1Size);
    // restart server2
    createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
    server1Size = getRegionSize(server1, regionName);
    int server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after server2 restarted: " + client1Size + "," + client2Size + "," + server1Size + ":" + server2Size);
    assertEquals(client2Size, server1Size);
    assertEquals(client2Size, server2Size);
    // close a server to re-run the test
    closeCache(server2);
    server1Size = getRegionSize(server1, regionName);
    client1.invoke(new CacheSerializableRunnable(title + "client1 does putAll again") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doPutAll(regionName, title + "again:", numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof PartitionOfflineException);
            }
        }
    });
    int new_server1Size = getRegionSize(server1, regionName);
    int new_client1Size = getRegionSize(client1, regionName);
    int new_client2Size = getRegionSize(client2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after re-run the putAll: " + new_client1Size + "," + new_client2Size + "," + new_server1Size);
    assertEquals(server1Size + numberOfEntries / 2, new_server1Size);
    assertEquals(client1Size + numberOfEntries / 2, new_client1Size);
    assertEquals(client2Size + numberOfEntries / 2, new_client2Size);
    // restart server2
    createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
    server1Size = getRegionSize(server1, regionName);
    server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after restart server2: " + server1Size + "," + server2Size);
    assertEquals(server1Size, server2Size);
    // add a cacheWriter for server to stop after created 15 keys
    server1.invoke(new CacheSerializableRunnable(title + "server1 execute P2P putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    // p2p putAll on PR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            // create keys
            try {
                doPutAll(regionName, title + "once again:", numberOfEntries);
                fail("Expected a CacheWriterException to be thrown by test");
            } catch (CacheWriterException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    new_server1Size = getRegionSize(server1, regionName);
    int new_server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after restart server2: " + new_server1Size + "," + new_server2Size);
    assertEquals(server1Size + 15, new_server1Size);
    assertEquals(server2Size + 15, new_server2Size);
    server1.invoke(removeExceptionTag1(expectedExceptions));
    server2.invoke(removeExceptionTag1(expectedExceptions));
    client1.invoke(removeExceptionTag1(expectedExceptions));
    client2.invoke(removeExceptionTag1(expectedExceptions));
    // Stop server
    stopBridgeServers(getCache());
}