Examples with CacheSerializableRunnable org.apache.geode.cache30.CacheSerializableRunnable used on opensource projects

Example 31 with CacheSerializableRunnable

use of org.apache.geode.cache30.CacheSerializableRunnable in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInPRSingleHop.

/**
   * Tests partial key putAll to 2 PR servers, because putting data at server side is different
   * between PR and LR. PR does it in postPutAll. This is a singlehop putAll test.
   */
@Test
public void testPartialKeyInPRSingleHop() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInPRSingleHop_";
    final int cacheWriterAllowedKeyNum = 16;
    final Host host = Host.getHost(0);
    final VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    final VM client1 = host.getVM(2);
    final VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, true, 0, "ds1");
    final int serverPort2 = createBridgeServer(server2, regionName, 0, true, 0, "ds1");
    createClient(client1, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true, false);
    createClient(client2, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    try {
        client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                region.getAttributesMutator().addCacheListener(new MyListener(false));
                region.registerInterest("ALL_KEYS");
                LogWriterUtils.getLogWriter().info("client2 registerInterest ALL_KEYS at " + region.getFullPath());
            }
        });
        client1.invoke(new CacheSerializableRunnable(title + "do some putAll to get ClientMetaData for future putAll") {

            @Override
            public void run2() throws CacheException {
                doPutAll(regionName, "key-", numberOfEntries);
            }
        });
        WaitCriterion waitForSizes = new WaitCriterion() {

            @Override
            public String description() {
                return "waiting for conditions to be met";
            }

            @Override
            public boolean done() {
                int c1Size = getRegionSize(client1, regionName);
                int c2Size = getRegionSize(client2, regionName);
                int s1Size = getRegionSize(server1, regionName);
                int s2Size = getRegionSize(server2, regionName);
                LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
                if (c1Size != numberOfEntries) {
                    LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                    return false;
                }
                if (c2Size != numberOfEntries) {
                    LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                    return false;
                }
                if (s1Size != numberOfEntries) {
                    LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                    return false;
                }
                if (s2Size != numberOfEntries) {
                    LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                    return false;
                }
                return true;
            }
        };
        Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
        server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                region.getAttributesMutator().addCacheListener(new MyListener(true));
            }
        });
        // add a listener that will close the cache at the 10th update
        final SharedCounter sc_server2 = new SharedCounter("server2");
        server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                region.getAttributesMutator().addCacheListener(new MyListener(server2, true, sc_server2, 10));
            }
        });
        AsyncInvocation async1 = client1.invokeAsync(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                region.getAttributesMutator().addCacheListener(new MyListener(false));
                // create keys
                try {
                    doPutAll(regionName, title, numberOfEntries);
                    fail("Expect ServerOperationException caused by PutAllParitialResultException");
                } catch (ServerOperationException soe) {
                    assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                }
            }
        });
        // server2 will closeCache after creating 10 keys
        ThreadUtils.join(async1, 30 * 1000);
        if (async1.exceptionOccurred()) {
            Assert.fail("putAll client threw an exception", async1.getException());
        }
        // restart server2
        System.out.println("restarting server 2");
        createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
        // Test Case1: Trigger singleHop putAll. Stop server2 in middle.
        // numberOfEntries/2 + X keys will be created at servers. i.e. X keys at server2,
        // numberOfEntries/2 keys at server1.
        // The client should receive a PartialResultException due to PartitionOffline
        // close a server to re-run the test
        closeCache(server2);
        client1.invoke(new CacheSerializableRunnable(title + "client1 does putAll again") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                // create keys
                try {
                    doPutAll(regionName, title + "again:", numberOfEntries);
                    fail("Expect ServerOperationException caused by PutAllParitialResultException");
                } catch (ServerOperationException soe) {
                    assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                }
            }
        });
        // Test Case 2: based on case 1, but this time, there should be no X keys
        // created on server2.
        // restart server2
        createBridgeServer(server2, regionName, serverPort2, true, 0, "ds1");
        // add a cacheWriter for server to fail putAll after it created cacheWriterAllowedKeyNum keys
        server1.invoke(new CacheSerializableRunnable(title + "server1 add cachewriter to throw exception after created some keys") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                region.getAttributesMutator().setCacheWriter(new MyWriter(cacheWriterAllowedKeyNum));
            }
        });
        client1.invoke(new CacheSerializableRunnable(title + "client1 does putAll once more") {

            @Override
            public void run2() throws CacheException {
                Region region = getRootRegion().getSubregion(regionName);
                // create keys
                try {
                    doPutAll(regionName, title + "once more:", numberOfEntries);
                    fail("Expect ServerOperationException caused by PutAllParitialResultException");
                } catch (ServerOperationException soe) {
                    assertTrue(soe.getMessage().contains(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(region.getFullPath())));
                }
            }
        });
    } finally {
        server1.invoke(removeExceptionTag1(expectedExceptions));
        server2.invoke(removeExceptionTag1(expectedExceptions));
        client1.invoke(removeExceptionTag1(expectedExceptions));
        client2.invoke(removeExceptionTag1(expectedExceptions));
        // Stop server
        stopBridgeServers(getCache());
    }
}

Example 32 with CacheSerializableRunnable

use of org.apache.geode.cache30.CacheSerializableRunnable in project geode by apache.

the class PutAllCSDUnitTest method testEventIdMisorderInPRSingleHop.

/**
   * Tests bug 41403: let 2 sub maps both failed with partial key applied. This is a singlehop
   * putAll test.
   */
@Test
public void testEventIdMisorderInPRSingleHop() throws CacheException, InterruptedException {
    final String title = "testEventIdMisorderInPRSingleHop_";
    final Host host = Host.getHost(0);
    final VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    final VM server3 = host.getVM(2);
    final VM client1 = host.getVM(3);
    final String regionName = getUniqueName();
    final int[] serverPorts = new int[3];
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    final SharedCounter sc_server1 = new SharedCounter("server1");
    final SharedCounter sc_server2 = new SharedCounter("server2");
    final SharedCounter sc_server3 = new SharedCounter("server3");
    final SharedCounter sc_client2 = new SharedCounter("client2");
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    serverPorts[0] = createBridgeServer(server1, regionName, 0, true, 0, null);
    serverPorts[1] = createBridgeServer(server2, regionName, 0, true, 0, null);
    serverPorts[2] = createBridgeServer(server3, regionName, 0, true, 0, null);
    createClient(client1, regionName, serverHost, serverPorts, -1, -1, false, true, true, false);
    {
        // Create local region
        Properties config = new Properties();
        config.setProperty(MCAST_PORT, "0");
        config.setProperty(LOCATORS, "");
        getSystem(config);
        // Create Region
        AttributesFactory factory = new AttributesFactory();
        factory.setScope(Scope.LOCAL);
        try {
            getCache();
            ClientServerTestCase.configureConnectionPool(factory, serverHost, serverPorts, true, -1, -1, null);
            createRegion(regionName, factory.create());
            assertNotNull(getRootRegion().getSubregion(regionName));
        } catch (CacheException ex) {
            Assert.fail("While creating Region on Edge", ex);
        }
    }
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    server3.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    addExceptionTag1(expectedExceptions);
    client1.invoke(new CacheSerializableRunnable(title + "do some putAll to get ClientMetaData for future putAll") {

        @Override
        public void run2() throws CacheException {
            doPutAll(regionName, "key-", numberOfEntries);
        }
    });
    // register interest and add listener
    MyListener myListener = new MyListener(false, sc_client2);
    Region region = getRootRegion().getSubregion(regionName);
    region.getAttributesMutator().addCacheListener(myListener);
    region.registerInterest("ALL_KEYS");
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region r = getRootRegion().getSubregion(regionName);
            r.getAttributesMutator().addCacheListener(new MyListener(server1, true, sc_server1, 10));
        }
    });
    server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region r = getRootRegion().getSubregion(regionName);
            r.getAttributesMutator().addCacheListener(new MyListener(server2, true, sc_server2, 10));
        }
    });
    server3.invoke(new CacheSerializableRunnable(title + "server3 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region r = getRootRegion().getSubregion(regionName);
            r.getAttributesMutator().addCacheListener(new MyListener(true, sc_server3));
        }
    });
    int client1Size = getRegionSize(client1, regionName);
    int server1Size = getRegionSize(server1, regionName);
    int server2Size = getRegionSize(server2, regionName);
    int server3Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes: " + client1Size + "," + server1Size + "," + server2Size + "," + server3Size);
    AsyncInvocation async1 = client1.invokeAsync(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region r = getRootRegion().getSubregion(regionName);
            r.getAttributesMutator().addCacheListener(new MyListener(false));
            doPutAll(regionName, title, numberOfEntries);
        }
    });
    // server1 and server2 will closeCache after created 10 keys
    ThreadUtils.join(async1, 30 * 1000);
    if (async1.exceptionOccurred()) {
        Assert.fail("Aync1 get exceptions:", async1.getException());
    }
    server3.invoke(new CacheSerializableRunnable(title + "server3 print counter") {

        @Override
        public void run2() throws CacheException {
            Region r = getRootRegion().getSubregion(regionName);
            MyListener l = (MyListener) r.getAttributes().getCacheListeners()[0];
            LogWriterUtils.getLogWriter().info("event counters : " + l.sc);
            assertEquals(numberOfEntries, l.sc.num_create_event);
            assertEquals(0, l.sc.num_update_event);
        }
    });
    LogWriterUtils.getLogWriter().info("event counters : " + myListener.sc);
    assertEquals(numberOfEntries, myListener.sc.num_create_event);
    assertEquals(0, myListener.sc.num_update_event);
    server1.invoke(removeExceptionTag1(expectedExceptions));
    server2.invoke(removeExceptionTag1(expectedExceptions));
    server3.invoke(removeExceptionTag1(expectedExceptions));
    client1.invoke(removeExceptionTag1(expectedExceptions));
    removeExceptionTag1(expectedExceptions);
    // Stop server
    stopBridgeServers(getCache());
}

Example 33 with CacheSerializableRunnable

use of org.apache.geode.cache30.CacheSerializableRunnable in project geode by apache.

the class PutAllCSDUnitTest method testEndPointSwitch.

/**
   * Tests while putAll timeout at endpoint1 and switch to endpoint2
   */
@Test
public void testEndPointSwitch() throws CacheException, InterruptedException {
    IgnoredException.addIgnoredException("Broken pipe");
    IgnoredException.addIgnoredException("Connection reset");
    IgnoredException.addIgnoredException("Unexpected IOException");
    final String title = "testEndPointSwitch:";
    disconnectAllFromDS();
    final Host host = Host.getHost(0);
    VM server1 = host.getVM(0);
    VM server2 = host.getVM(1);
    VM client1 = host.getVM(2);
    VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set notifyBySubscription=true to test register interest
    final int serverPort1 = createBridgeServer(server1, regionName, 0, false, 0, null);
    final int serverPort2 = createBridgeServer(server2, regionName, 0, false, 0, null);
    createBridgeClient(client1, regionName, serverHost, new int[] { serverPort1, serverPort2 }, 1, -1, true);
    createClient(client2, regionName, serverHost, new int[] { serverPort2, serverPort1 }, 1, -1, false, true, true);
    // only add slow listener to server1, because we wish it to succeed
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    // only register interest on client2
    client2.invoke(new CacheSerializableRunnable(title + "client2 registerInterest") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client2 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    // Execute client1 putAll
    client1.invoke(new CacheSerializableRunnable(title + "putAll from client1") {

        @Override
        public void run2() throws CacheException {
            try {
                doPutAll(regionName, title, testEndPointSwitchNumber);
            } catch (Exception e) {
                LogWriterUtils.getLogWriter().info(title + "Expected SocketTimeOut" + e.getMessage());
            }
        }
    });
    // verify client 2 for all keys
    client2.invoke(new CacheSerializableRunnable(title + "verify Bridge client2 for keys arrived finally") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            waitTillNotify(lockObject3, 100000, (region.size() == testEndPointSwitchNumber));
            assertEquals(testEndPointSwitchNumber, region.size());
        }
    });
    // clean up
    // Stop server
    stopBridgeServers(getCache());
}

Example 34 with CacheSerializableRunnable

use of org.apache.geode.cache30.CacheSerializableRunnable in project geode by apache.

the class PutAllCSDUnitTest method testPartialKeyInPRSingleHopWithRedundency.

/**
   * Set redundancy=1 to see if retry succeeded after PRE This is a singlehop putAll test.
   */
@Test
public void testPartialKeyInPRSingleHopWithRedundency() throws CacheException, InterruptedException {
    final String title = "testPartialKeyInPRSingleHopWithRedundency_";
    int client1Size;
    int client2Size;
    int server1Size;
    int server2Size;
    final Host host = Host.getHost(0);
    final VM server1 = host.getVM(0);
    final VM server2 = host.getVM(1);
    final VM client1 = host.getVM(2);
    final VM client2 = host.getVM(3);
    final String regionName = getUniqueName();
    final String serverHost = NetworkUtils.getServerHostName(server1.getHost());
    // set <true, false> means <PR=true, notifyBySubscription=false> to test local-invalidates
    final int serverPort1 = createBridgeServer(server1, regionName, 0, true, 1, "ds1");
    final int serverPort2 = createBridgeServer(server2, regionName, 0, true, 1, "ds1");
    createClient(client1, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true, false);
    createClient(client2, regionName, serverHost, new int[] { serverPort1, serverPort2 }, -1, -1, false, false, true);
    server1.invoke(addExceptionTag1(expectedExceptions));
    server2.invoke(addExceptionTag1(expectedExceptions));
    client1.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(addExceptionTag1(expectedExceptions));
    client2.invoke(new CacheSerializableRunnable(title + "client2 add listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            LogWriterUtils.getLogWriter().info("client2 registerInterest ALL_KEYS at " + region.getFullPath());
        }
    });
    client1.invoke(new CacheSerializableRunnable(title + "do some putAll to get ClientMetaData for future putAll") {

        @Override
        public void run2() throws CacheException {
            doPutAll(regionName, "key-", numberOfEntries);
        }
    });
    WaitCriterion waitForSizes = new WaitCriterion() {

        @Override
        public String description() {
            return "waiting for conditions to be met";
        }

        @Override
        public boolean done() {
            int c1Size = getRegionSize(client1, regionName);
            int c2Size = getRegionSize(client2, regionName);
            int s1Size = getRegionSize(server1, regionName);
            int s2Size = getRegionSize(server2, regionName);
            LogWriterUtils.getLogWriter().info("region sizes: " + c1Size + "," + c2Size + "," + s1Size + "," + s2Size);
            if (c1Size != numberOfEntries) {
                LogWriterUtils.getLogWriter().info("waiting for client1 to get all updates");
                return false;
            }
            if (c2Size != numberOfEntries) {
                LogWriterUtils.getLogWriter().info("waiting for client2 to get all updates");
                return false;
            }
            if (s1Size != numberOfEntries) {
                LogWriterUtils.getLogWriter().info("waiting for server1 to get all updates");
                return false;
            }
            if (s2Size != numberOfEntries) {
                LogWriterUtils.getLogWriter().info("waiting for server2 to get all updates");
                return false;
            }
            return true;
        }
    };
    Wait.waitForCriterion(waitForSizes, 10000, 1000, true);
    client1Size = getRegionSize(client1, regionName);
    client2Size = getRegionSize(client2, regionName);
    server1Size = getRegionSize(server1, regionName);
    server2Size = getRegionSize(server2, regionName);
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    final SharedCounter sc_server2 = new SharedCounter("server2");
    server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(server2, true, sc_server2, 10));
        }
    });
    AsyncInvocation async1 = client1.invokeAsync(new CacheSerializableRunnable(title + "client1 add listener and putAll") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(false));
            region.registerInterest("ALL_KEYS");
            // create keys
            doPutAll(regionName, title, numberOfEntries);
        }
    });
    // server2 will closeCache after created 10 keys
    ThreadUtils.join(async1, 30 * 1000);
    if (async1.exceptionOccurred()) {
        Assert.fail("Aync1 get exceptions:", async1.getException());
    }
    client1Size = getRegionSize(client1, regionName);
    // client2Size maybe more than client1Size
    client2Size = getRegionSize(client2, regionName);
    server1Size = getRegionSize(server1, regionName);
    // putAll should succeed after retry
    LogWriterUtils.getLogWriter().info("region sizes: " + client1Size + "," + client2Size + "," + server1Size);
    assertEquals(server1Size, client1Size);
    assertEquals(server1Size, client2Size);
    // restart server2
    createBridgeServer(server2, regionName, serverPort2, true, 1, "ds1");
    server1Size = getRegionSize(server1, regionName);
    server2Size = getRegionSize(server2, regionName);
    LogWriterUtils.getLogWriter().info("region sizes after server2 restarted: " + client1Size + "," + client2Size + "," + server1Size);
    assertEquals(client2Size, server1Size);
    assertEquals(client2Size, server2Size);
    // close a server to re-run the test
    closeCache(server2);
    server1Size = getRegionSize(server1, regionName);
    client1.invoke(new CacheSerializableRunnable(title + "client1 does putAll again") {

        @Override
        public void run2() throws CacheException {
            doPutAll(regionName, title + "again:", numberOfEntries);
        }
    });
    int new_server1Size = getRegionSize(server1, regionName);
    int new_client1Size = getRegionSize(client1, regionName);
    int new_client2Size = getRegionSize(client2, regionName);
    // putAll should succeed, all the numbers should match
    LogWriterUtils.getLogWriter().info("region sizes after re-run the putAll: " + new_client1Size + "," + new_client2Size + "," + new_server1Size);
    assertEquals(new_server1Size, new_client1Size);
    assertEquals(new_server1Size, new_client2Size);
    // Stop server
    stopBridgeServers(getCache());
}

Example 35 with CacheSerializableRunnable

use of org.apache.geode.cache30.CacheSerializableRunnable in project geode by apache.

the class PutAllCSDUnitTest method testTX.

/**
   * Test TX for putAll. There's no TX for c/s. We only test P2P This is disabled because putAll in
   * TX is disabled.
   */
@Ignore("TODO: test is disabled")
@Test
public void testTX() throws CacheException, InterruptedException {
    final String title = "testTX:";
    disconnectAllFromDS();
    final Host host = Host.getHost(0);
    VM server1 = host.getVM(0);
    VM server2 = host.getVM(1);
    final String regionName = getUniqueName();
    final int serverPort1 = createBridgeServer(server1, regionName, 0, false, 0, null);
    final int serverPort2 = createBridgeServer(server2, regionName, 0, false, 0, null);
    // final String serverHost = getServerHostName(server1.getHost());
    // set notifyBySubscription=true to test register interest
    // add slow listener
    server1.invoke(new CacheSerializableRunnable(title + "server1 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    server2.invoke(new CacheSerializableRunnable(title + "server2 add slow listener") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            region.getAttributesMutator().addCacheListener(new MyListener(true));
        }
    });
    // TX1: server1 do a putAll
    AsyncInvocation async1 = server1.invokeAsync(new CacheSerializableRunnable(title + "TX1: async putAll from server1") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // Get JNDI(Java Naming and Directory interface) context
            // CacheTransactionManager tx = getCache().getCacheTransactionManager();
            LinkedHashMap map = new LinkedHashMap();
            // tx.begin();
            for (int i = 0; i < numberOfEntries; i++) {
                // region.put("key-"+i, new TestObject(i));
                map.put("key-" + i, new TestObject(i));
            }
            region.putAll(map, "putAllCallback");
            try {
                LogWriterUtils.getLogWriter().info("before commit TX1");
                // tx.commit();
                LogWriterUtils.getLogWriter().info("TX1 committed");
            } catch (CommitConflictException e) {
                LogWriterUtils.getLogWriter().info("TX1 rollbacked");
            }
        }
    });
    // we have to pause a while to let TX1 finish earlier
    Wait.pause(500);
    // TX2: server2 do a putAll
    AsyncInvocation async2 = server2.invokeAsync(new CacheSerializableRunnable(title + "TX2: async putAll from server2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // Get JNDI(Java Naming and Directory interface) context
            // CacheTransactionManager tx = getCache().getCacheTransactionManager();
            LinkedHashMap map = new LinkedHashMap();
            // tx.begin();
            for (int i = 0; i < numberOfEntries; i++) {
                // region.put("key-"+i, new TestObject(i + numberOfEntries));
                map.put("key-" + i, new TestObject(i + numberOfEntries));
            }
            region.putAll(map, "putAllCallback");
            try {
                LogWriterUtils.getLogWriter().info("before commit TX2");
                // tx.commit();
                LogWriterUtils.getLogWriter().info("TX2 committed");
            } catch (CommitConflictException e) {
                LogWriterUtils.getLogWriter().info("TX2 rollbacked");
            }
        }
    });
    // TX3: server2 do a putAll in another thread
    AsyncInvocation async3 = server2.invokeAsync(new CacheSerializableRunnable(title + "TX3: async putAll from server2") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            // Get JNDI(Java Naming and Directory interface) context
            // CacheTransactionManager tx = getCache().getCacheTransactionManager();
            LinkedHashMap map = new LinkedHashMap();
            // tx.begin();
            for (int i = 0; i < numberOfEntries; i++) {
                // region.put("key-"+i, new TestObject(i+numberOfEntries*2));
                map.put("key-" + i, new TestObject(i + numberOfEntries * 2));
            }
            region.putAll(map, "putAllCallback");
            try {
                LogWriterUtils.getLogWriter().info("before commit TX3");
                // tx.commit();
                LogWriterUtils.getLogWriter().info("TX3 committed");
            } catch (CommitConflictException e) {
                LogWriterUtils.getLogWriter().info("TX3 rollbacked");
            }
        }
    });
    ThreadUtils.join(async1, 30 * 1000);
    ThreadUtils.join(async2, 30 * 1000);
    ThreadUtils.join(async3, 30 * 1000);
    // verify server 2 for asyn keys
    server2.invoke(new CacheSerializableRunnable(title + "verify Bridge server2 for keys") {

        @Override
        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            assertEquals(numberOfEntries, region.size());
            int tx_no = 0;
            for (int i = 0; i < numberOfEntries; i++) {
                TestObject obj = (TestObject) region.getEntry("key-" + i).getValue();
                if (tx_no == 0) {
                    // only check which tx took control once
                    if (obj.getPrice() == i) {
                        tx_no = 1;
                    } else if (obj.getPrice() == i + numberOfEntries) {
                        tx_no = 2;
                    } else if (obj.getPrice() == i + numberOfEntries * 2) {
                        tx_no = 3;
                    }
                    LogWriterUtils.getLogWriter().info("Verifying TX:" + tx_no);
                }
                if (tx_no == 1) {
                    assertEquals(i, obj.getPrice());
                } else if (tx_no == 2) {
                    assertEquals(i + numberOfEntries, obj.getPrice());
                } else {
                    assertEquals(i + numberOfEntries * 2, obj.getPrice());
                }
            }
        }
    });
    // clean up
    // Stop server
    stopBridgeServers(getCache());
}