Examples with ServerOperationException org.apache.geode.cache.client.ServerOperationException used on opensource projects

Example 11 with ServerOperationException

use of org.apache.geode.cache.client.ServerOperationException in project geode by apache.

the class Bug36805DUnitTest method registerInterest.

public static void registerInterest() {
    cache.getLogger().info("<ExpectedException action=add>" + "RegionDestroyedException" + "</ExpectedException>");
    try {
        Region r = cache.getRegion("/" + regionName);
        assertNotNull(r);
        List listOfKeys = new ArrayList();
        listOfKeys.add("key-1");
        listOfKeys.add("key-2");
        listOfKeys.add("key-3");
        listOfKeys.add("key-4");
        listOfKeys.add("key-5");
        r.registerInterest(listOfKeys);
        fail("expected RegionDestroyedException");
    } catch (ServerOperationException expected) {
        assertEquals(RegionDestroyedException.class, expected.getCause().getClass());
    } finally {
        cache.getLogger().info("<ExpectedException action=remove>" + "RegionDestroyedException" + "</ExpectedException>");
    }
}

Example 12 with ServerOperationException

use of org.apache.geode.cache.client.ServerOperationException in project geode by apache.

the class ClassNotFoundExceptionDUnitTest method doTest.

public void doTest(final ObjectFactory objectFactory) throws InterruptedException {
    IgnoredException.addIgnoredException("SerializationException");
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    VM vm3 = host.getVM(3);
    int port1 = createServerRegion(vm0);
    int port2 = createServerRegion(vm1);
    createClientRegion(vm2, port1);
    createClientRegion(vm3, port2);
    SerializableRunnable putKey = new SerializableRunnable() {

        public void run() {
            Region region = getCache().getRegion("testSimplePdx");
            region.put("a", "b");
            region.put("b", "b");
            for (int i = 0; i < 10; i++) {
                region.put(i, i);
            }
            if (!region.containsKey("test")) {
                region.put("test", objectFactory.get());
            }
            try {
                region.put(objectFactory.get(), objectFactory.get());
                fail("Should have received an exception");
            } catch (SerializationException expected) {
            // ok
            } catch (ServerOperationException expected) {
                if (!(expected.getCause() instanceof SerializationException) && !(expected.getCause() instanceof ClassNotFoundException)) {
                    throw expected;
                }
            }
        // try {
        // region.replace("test", objectFactory.get(), objectFactory.get());
        // fail("Should have received an exception");
        // } catch(SerializationException expected) {
        // //ok
        // } catch(ServerOperationException expected) {
        // if(!(expected.getCause() instanceof SerializationException) && !(expected.getCause()
        // instanceof ClassNotFoundException)) {
        // throw expected;
        // }
        // }
        }
    };
    SerializableRunnable getValue = new SerializableRunnable() {

        public void run() {
            Region region = getCache().getRegion("testSimplePdx");
            try {
                assertNotNull(region.get("test"));
                fail("Should have received an exception");
            } catch (SerializationException expected) {
            // ok
            } catch (ServerOperationException expected) {
                if (!(expected.getCause() instanceof SerializationException) && !(expected.getCause() instanceof ClassNotFoundException)) {
                    throw expected;
                }
            }
        }
    };
    SerializableRunnable registerInterest = new SerializableRunnable() {

        public void run() {
            Region region = getCache().getRegion("testSimplePdx");
            try {
                ArrayList keys = new ArrayList();
                for (int i = 0; i < 1000; i++) {
                    keys.add(i);
                }
                keys.add("test");
                region.getAll(keys);
                fail("Should have received an exception");
            } catch (SerializationException expected) {
                System.out.println("hi");
            // ok
            } catch (ServerOperationException expected) {
                if (!(expected.getCause() instanceof SerializationException) && !(expected.getCause() instanceof ClassNotFoundException)) {
                    throw expected;
                }
            }
        }
    };
    vm2.invoke(putKey);
    vm1.invoke(getValue);
    vm3.invoke(getValue);
    vm3.invoke(registerInterest);
    vm1.invoke(putKey);
}

Example 13 with ServerOperationException

use of org.apache.geode.cache.client.ServerOperationException in project geode by apache.

the class SecurityTestUtils method doFunctionExecuteP.

private static void doFunctionExecuteP(final int multiUserIndex, final Function function, int expectedResult, final String method) {
    Region region = null;
    try {
        if (multiUserAuthMode) {
            region = proxyCaches[multiUserIndex].getRegion(REGION_NAME);
        } else {
            region = getCache().getRegion(REGION_NAME);
        }
        assertNotNull(region);
    } catch (Exception ex) {
        if (expectedResult == OTHER_EXCEPTION) {
            getLogWriter().info("Got expected exception when executing function: " + ex);
        } else {
            fail("Got unexpected exception when executing function", ex);
        }
    }
    try {
        FunctionService.registerFunction(function);
        Execution execution = null;
        if ("region".equals(method)) {
            execution = FunctionService.onRegion(region);
        } else if ("server".equals(method)) {
            if (multiUserAuthMode) {
                execution = FunctionService.onServer(proxyCaches[multiUserIndex]);
            } else {
                execution = FunctionService.onServer(pool);
            }
        } else {
            // if ("servers".equals(method)) {
            if (multiUserAuthMode) {
                execution = FunctionService.onServers(proxyCaches[multiUserIndex]);
            } else {
                execution = FunctionService.onServers(pool);
            }
        }
        execution.execute(function.getId());
        if (expectedResult != NO_EXCEPTION) {
            fail("Expected a NotAuthorizedException while executing function");
        }
    } catch (NoAvailableServersException ex) {
        if (expectedResult == NO_AVAILABLE_SERVERS) {
            getLogWriter().info("Got expected NoAvailableServers when executing function: " + ex.getCause());
        } else {
            fail("Got unexpected exception when executing function", ex);
        }
    } catch (ServerConnectivityException ex) {
        if ((expectedResult == NOTAUTHZ_EXCEPTION) && (ex.getCause() instanceof NotAuthorizedException)) {
            getLogWriter().info("Got expected NotAuthorizedException when executing function: " + ex.getCause());
        } else if (expectedResult == OTHER_EXCEPTION) {
            getLogWriter().info("Got expected exception when executing function: " + ex);
        } else {
            fail("Got unexpected exception when executing function", ex);
        }
    } catch (FunctionException ex) {
        // ServerOperationException AND cause.cause is NotAuthorizedException))
        if (expectedResult == NOTAUTHZ_EXCEPTION && (ex.getCause() instanceof NotAuthorizedException || (ex.getCause() instanceof ServerOperationException && ex.getCause().getCause() instanceof NotAuthorizedException))) {
            getLogWriter().info("Got expected NotAuthorizedException when executing function: " + ex.getCause());
        } else if (expectedResult == OTHER_EXCEPTION) {
            getLogWriter().info("Got expected exception when executing function: " + ex);
        } else {
            fail("Got unexpected exception when executing function", ex);
        }
    } catch (Exception ex) {
        if (expectedResult == OTHER_EXCEPTION) {
            getLogWriter().info("Got expected exception when executing function: " + ex);
        } else {
            fail("Got unexpected exception when executing function", ex);
        }
    }
}

Example 14 with ServerOperationException

use of org.apache.geode.cache.client.ServerOperationException in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInLocalRegion.

/**
   * Tests partial key putAll and removeAll to 2 servers with local region
   */
@Test
public void testPartialKeyInLocalRegion() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInLocalRegion:";
    final Host host = Host.getHost(0);
    final VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    final VM client1 = host.getVM(2);
    final VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, false, 0, null);
    final int serverPort2 = createBridgeServer(server2, regionName, 0, false, 0, null);
    createClient(client1, regionName, serverHost, new int[] { serverPort1 }, -1, -1, false, true, true);
    createClient(client2, regionName, serverHost, new int[] { serverPort1 }, -1, -1, false, true, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client1 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doPutAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof RuntimeException);
                assertTrue(soe.getCause().getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                    return false;
                }
                if (c2Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                    return false;
                }
                if (s1Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                    return false;
                }
                if (s2Size != 15) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    int server1Size = getRegionSize(server1, regionName);
    int server2Size = getRegionSize(server1, regionName);
    // reset cacheWriter's count to allow another 15 keys to be created
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    // p2p putAll on DR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            // create keys
            try {
                doPutAll(regionName, title + "again:", numberOfEntries);
                fail("Expect original RuntimeException caused by cacheWriter");
            } catch (RuntimeException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    server2Size = getRegionSize(server1, regionName);
    assertEquals(server1Size + 15, server2Size);
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                    return false;
                }
                if (c2Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                    return false;
                }
                if (s1Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                    return false;
                }
                if (s2Size != 15 * 2) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    // now do a removeAll that is not allowed to remove everything
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // server triggers exception after destroying 5 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(5));
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "client1 removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doRemoveAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_RemoveAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof RuntimeException);
                assertTrue(soe.getCause().getMessage().contains("Triggered exception as planned, destroyed 5 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15 - 5) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all destroys");
                    return false;
                }
                if (c2Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all destroys");
                    return false;
                }
                if (s1Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all destroys");
                    return false;
                }
                if (s2Size != (15 * 2) - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all destroys");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    // reset cacheWriter's count to allow another 5 keys to be destroyed
    server1.invoke(new CacheSerializableRunnable(title + "server1 add cacheWriter") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // server triggers exception after destroying 5 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(5));
        }
    });
    // p2p putAll on DR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and removeAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            // create keys
            try {
                doRemoveAll(regionName, title + "again:", numberOfEntries);
                fail("Expect original RuntimeException caused by cacheWriter");
            } catch (RuntimeException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, destroyed 5 keys"));
            }
        }
    });
    {
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != 15 - 5) {
                    // client 1 did not register interest
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all destroys");
                    return false;
                }
                if (c2Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all destroys");
                    return false;
                }
                if (s1Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all destroys");
                    return false;
                }
                if (s2Size != (15 * 2) - 5 - 5) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all destroys");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    }
    server1.invoke(removeExceptionTag1(expectedExceptions));
    server2.invoke(removeExceptionTag1(expectedExceptions));
    client1.invoke(removeExceptionTag1(expectedExceptions));
    client2.invoke(removeExceptionTag1(expectedExceptions));
    // Stop server
    stopBridgeServers(getCache());
}

Example 15 with ServerOperationException

use of org.apache.geode.cache.client.ServerOperationException in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInPR.

/**
   * Tests partial key putAll to 2 PR servers, because putting data at server side is different
   * between PR and LR. PR does it in postPutAll. It's not running in singleHop putAll
   */
@Test
public void testPartialKeyInPR() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInPR:";
    final Host host = Host.getHost(0);
    VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    VM client1 = host.getVM(2);
    VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, true, 0, "ds1");
    final int serverPort2 = createBridgeServer(server2, regionName, 0, true, 0, "ds1");
    createClient(client1, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    createClient(client2, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    final SharedCounter sc_server2 = new SharedCounter("server2");
    server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(server2, true, sc_server2, 10));
        }
    });
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client2 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    AsyncInvocation async1 = client1.invokeAsync(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            // create keys
            try {
                doPutAll(regionName, title, numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                if (!(soe.getCause() instanceof PartitionOfflineException)) {
                    throw soe;
                }
                if (!soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath()))) {
                    throw soe;
                }
            }
        }
    });
    // server2 will closeCache after created 10 keys
    ThreadUtils.join(async1, 30 * 1000);
    if (async1.exceptionOccurred()) {
        Assert.fail("Aync1 get exceptions:", async1.getException());
    }
    int client1Size = getRegionSize(client1, regionName);
    // client2Size maybe more than client1Size
    int client2Size = getRegionSize(client2, regionName);
    int server1Size = getRegionSize(server1, regionName);
    LogWriterUtils.getLogWriter().info("region sizes: " + client1Size + "," + client2Size + "," + server1Size);
    // restart server2
    createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
    server1Size = getRegionSize(server1, regionName);
    int server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after server2 restarted: " + client1Size + "," + client2Size + "," + server1Size + ":" + server2Size);
    assertEquals(client2Size, server1Size);
    assertEquals(client2Size, server2Size);
    // close a server to re-run the test
    closeCache(server2);
    server1Size = getRegionSize(server1, regionName);
    client1.invoke(new CacheSerializableRunnable(title + "client1 does putAll again") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // create keys
            try {
                doPutAll(regionName, title + "again:", numberOfEntries);
                fail("Expect ServerOperationException caused by PutAllParitialResultException");
            } catch (ServerOperationException soe) {
                assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                assertTrue(soe.getCause() instanceof PartitionOfflineException);
            }
        }
    });
    int new_server1Size = getRegionSize(server1, regionName);
    int new_client1Size = getRegionSize(client1, regionName);
    int new_client2Size = getRegionSize(client2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after re-run the putAll: " + new_client1Size + "," + new_client2Size + "," + new_server1Size);
    assertEquals(server1Size + numberOfEntries / 2, new_server1Size);
    assertEquals(client1Size + numberOfEntries / 2, new_client1Size);
    assertEquals(client2Size + numberOfEntries / 2, new_client2Size);
    // restart server2
    createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
    server1Size = getRegionSize(server1, regionName);
    server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after restart server2: " + server1Size + "," + server2Size);
    assertEquals(server1Size, server2Size);
    // add a cacheWriter for server to stop after created 15 keys
    server1.invoke(new CacheSerializableRunnable(title + "server1 execute P2P putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // let the server to trigger exception after created 15 keys
            region.getAttributesMutator().setCacheWriter(new MyWriter(15));
        }
    });
    // p2p putAll on PR and expect exception
    server2.invoke(new CacheSerializableRunnable(title + "server2 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            // create keys
            try {
                doPutAll(regionName, title + "once again:", numberOfEntries);
                fail("Expected a CacheWriterException to be thrown by test");
            } catch (CacheWriterException rte) {
                assertTrue(rte.getMessage().contains("Triggered exception as planned, created 15 keys"));
            }
        }
    });
    new_server1Size = getRegionSize(server1, regionName);
    int new_server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after restart server2: " + new_server1Size + "," + new_server2Size);
    assertEquals(server1Size + 15, new_server1Size);
    assertEquals(server2Size + 15, new_server2Size);
    server1.invoke(removeExceptionTag1(expectedExceptions));
    server2.invoke(removeExceptionTag1(expectedExceptions));
    client1.invoke(removeExceptionTag1(expectedExceptions));
    client2.invoke(removeExceptionTag1(expectedExceptions));
    // Stop server
    stopBridgeServers(getCache());
}