Examples with Execution org.apache.geode.cache.execute.Execution used on opensource projects

Example 6 with Execution

use of org.apache.geode.cache.execute.Execution in project geode by apache.

the class FireAndForgetFunctionOnAllServersDUnitTest method testFireAndForgetFunctionOnAllServers.

@Test
public void testFireAndForgetFunctionOnAllServers() {
    // Test case for Executing a fire-and-forget function on all servers as opposed to only
    // executing on the ones the
    // client is currently connected to.
    Host host = Host.getHost(0);
    VM locator = host.getVM(0);
    VM server1 = host.getVM(1);
    VM server2 = host.getVM(2);
    VM client = host.getVM(3);
    final int locatorPort = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
    final String locatorHost = NetworkUtils.getServerHostName(host);
    // Step 1. Start a locator and one cache server.
    locator.invoke("Start Locator", () -> startLocator(locatorHost, locatorPort, ""));
    String locString = getLocatorString(host, locatorPort);
    // Step 2. Start a server and create a replicated region "R1".
    server1.invoke("Start BridgeServer", () -> startBridgeServer(new String[] { "R1" }, locString, new String[] { "R1" }));
    // Step 3. Create a client cache with pool mentioning locator.
    client.invoke("create client cache and pool mentioning locator", () -> {
        ClientCacheFactory ccf = new ClientCacheFactory();
        ccf.addPoolLocator(locatorHost, locatorPort);
        ClientCache cache = ccf.create();
        Pool pool1 = PoolManager.createFactory().addLocator(locatorHost, locatorPort).setServerGroup("R1").create("R1");
        Region region1 = cache.createClientRegionFactory(ClientRegionShortcut.PROXY).setPoolName("R1").create("R1");
        // Step 4. Execute the function to put DistributedMemberID into above created replicated
        // region.
        Function function = new FireAndForgetFunctionOnAllServers();
        FunctionService.registerFunction(function);
        PoolImpl pool = (PoolImpl) pool1;
        await().atMost(60, SECONDS).until(() -> Assert.assertEquals(1, pool.getCurrentServers().size()));
        String regionName = "R1";
        Execution dataSet = FunctionService.onServers(pool1);
        dataSet.setArguments(regionName).execute(function);
        // Using Awatility, if the condition is not met during the timeout, a
        // ConditionTimeoutException will be thrown. This makes analyzing the failure much simpler
        // Step 5. Assert for the region keyset size with 1.
        await().atMost(60, SECONDS).until(() -> Assert.assertEquals(1, region1.keySetOnServer().size()));
        region1.clear();
        // Step 6. Start another server mentioning locator and create a replicated region "R1".
        server2.invoke("Start BridgeServer", () -> startBridgeServer(new String[] { "R1" }, locString, new String[] { "R1" }));
        // Step 7. Execute the same function to put DistributedMemberID into above created replicated
        // region.
        await().atMost(60, SECONDS).until(() -> Assert.assertEquals(1, pool.getCurrentServers().size()));
        dataSet = FunctionService.onServers(pool1);
        dataSet.setArguments(regionName).execute(function);
        await().atMost(60, SECONDS).until(() -> Assert.assertEquals(2, pool.getCurrentServers().size()));
        // Using Awatility, if the condition is not met during the timeout, a
        // ConditionTimeoutException will be thrown. This makes analyzing the failure much simpler
        // Step 8. Assert for the region keyset size with 2, since above function was executed on 2
        // servers.
        await().atMost(60, SECONDS).until(() -> {
            Assert.assertEquals(2, region1.keySetOnServer().size());
        });
        region1.clear();
        // Step 8.Stop one of the servers.
        server1.invoke("Stop BridgeServer", () -> stopBridgeMemberVM(server1));
        // Step 9. Execute the same function to put DistributedMemberID into above created replicated
        // region.
        dataSet = FunctionService.onServers(pool1);
        dataSet.setArguments(regionName).execute(function);
        await().atMost(60, SECONDS).until(() -> Assert.assertEquals(1, pool.getCurrentServers().size()));
        // Using Awatility, if the condition is not met during the timeout, a
        // ConditionTimeoutException will be thrown. This makes analyzing the failure much simpler
        // Step 10. Assert for the region keyset size with 1, since only one server was running.
        await().atMost(60, SECONDS).until(() -> {
            Assert.assertEquals(1, region1.keySetOnServer().size());
        });
        region1.clear();
        return null;
    });
}

Example 7 with Execution

use of org.apache.geode.cache.execute.Execution in project geode by apache.

the class RollbackFunction method execute.

public void execute(FunctionContext context) {
    Cache cache = CacheFactory.getAnyInstance();
    TXId txId = null;
    try {
        txId = (TXId) context.getArguments();
    } catch (ClassCastException e) {
        logger.info("RollbackFunction should be invoked with a TransactionId as an argument i.e. setArguments(txId).execute(function)");
        throw e;
    }
    DistributedMember member = txId.getMemberId();
    Boolean result = false;
    final boolean isDebugEnabled = logger.isDebugEnabled();
    if (cache.getDistributedSystem().getDistributedMember().equals(member)) {
        if (isDebugEnabled) {
            logger.debug("RollbackFunction: for transaction: {} rolling back locally", txId);
        }
        CacheTransactionManager txMgr = cache.getCacheTransactionManager();
        if (txMgr.tryResume(txId)) {
            if (isDebugEnabled) {
                logger.debug("RollbackFunction: resumed transaction: {}", txId);
            }
            txMgr.rollback();
            result = true;
        }
    } else {
        ArrayList args = new ArrayList();
        args.add(txId);
        args.add(NestedTransactionFunction.ROLLBACK);
        Execution ex = FunctionService.onMember(member).setArguments(args);
        if (isDebugEnabled) {
            logger.debug("RollbackFunction: for transaction: {} executing NestedTransactionFunction on member: {}", txId, member);
        }
        try {
            List list = (List) ex.execute(new NestedTransactionFunction()).getResult();
            result = (Boolean) list.get(0);
        } catch (FunctionException fe) {
            throw new TransactionDataNodeHasDepartedException("Could not Rollback on member:" + member);
        }
    }
    if (isDebugEnabled) {
        logger.debug("RollbackFunction: for transaction: {} returning result: {}", txId, result);
    }
    context.getResultSender().lastResult(result);
}

Example 8 with Execution

use of org.apache.geode.cache.execute.Execution in project geode by apache.

the class FunctionServiceStatsDUnitTest method testP2PMembersFunctionExecutionStats.

/**
   * Test the execution of function on all memebers haveResults = true
   * 
   * member1 calls for the function executions sp the results received on memeber 1 should be equal
   * to the no of function execution calls. Function Execution should happen on all other members
   * too. so the no of function execution calls and no of function executions completed should be
   * equal tio the no of functions from member 1
   */
@Test
public void testP2PMembersFunctionExecutionStats() throws Exception {
    Host host = Host.getHost(0);
    VM member1 = host.getVM(0);
    VM member2 = host.getVM(1);
    VM member3 = host.getVM(2);
    VM member4 = host.getVM(3);
    SerializableCallable connectToDistributedSystem = new SerializableCallable("connectToDistributedSystem") {

        public Object call() throws Exception {
            Properties props = new Properties();
            try {
                ds = getSystem(props);
                assertNotNull(ds);
            } catch (Exception e) {
                Assert.fail("Failed while creating the Distribued System", e);
            }
            return Boolean.TRUE;
        }
    };
    member1.invoke(connectToDistributedSystem);
    member2.invoke(connectToDistributedSystem);
    member3.invoke(connectToDistributedSystem);
    member4.invoke(connectToDistributedSystem);
    member1.invoke(initializeStats);
    member2.invoke(initializeStats);
    member3.invoke(initializeStats);
    member4.invoke(initializeStats);
    final int noOfMembers = 1;
    final Function inlineFunction = new FunctionAdapter() {

        public void execute(FunctionContext context) {
            if (context.getArguments() instanceof String) {
                context.getResultSender().lastResult("Success");
            } else {
                context.getResultSender().lastResult("Failure");
            }
        }

        public String getId() {
            return getClass().getName();
        }

        public boolean hasResult() {
            return true;
        }
    };
    member1.invoke(new SerializableCallable("excuteOnMembers_InlineFunction") {

        public Object call() throws Exception {
            assertNotNull(ds);
            Execution memberExecution = null;
            DistributedMember localmember = ds.getDistributedMember();
            memberExecution = FunctionService.onMember(localmember);
            memberExecution.setArguments("Key");
            try {
                ResultCollector rc = memberExecution.execute(inlineFunction);
                int size = ((List) rc.getResult()).size();
                resultReceived_Aggregate += size;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionsCompleted_Aggregate++;
                resultReceived_Inline += size;
                noOfExecutionCalls_Inline++;
                noOfExecutionsCompleted_Inline++;
            } catch (Exception e) {
                LogWriterUtils.getLogWriter().info("Exception Occurred : " + e.getMessage());
                e.printStackTrace();
                Assert.fail("Test failed", e);
            }
            return Boolean.TRUE;
        }
    });
    member1.invoke(new SerializableCallable("checkFunctionExecutionStatsForMember1") {

        public Object call() throws Exception {
            FunctionServiceStats functionServiceStats = ds.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            assertEquals(noOfExecutionCalls_Aggregate, functionServiceStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Aggregate, functionServiceStats.getFunctionExecutionsCompleted());
            assertEquals(resultReceived_Aggregate, functionServiceStats.getResultsReceived());
            FunctionStats functionStats = FunctionStats.getFunctionStats(inlineFunction.getId(), ds);
            assertEquals(noOfExecutionCalls_Inline, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Inline, functionStats.getFunctionExecutionsCompleted());
            assertEquals(resultReceived_Inline, functionStats.getResultsReceived());
            return Boolean.TRUE;
        }
    });
    SerializableCallable checkFunctionExecutionStatsForOtherMember = new SerializableCallable("checkFunctionExecutionStatsForOtherMember") {

        public Object call() throws Exception {
            FunctionServiceStats functionServiceStats = ds.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            // One function Execution took place on there members
            // noOfExecutionCalls_Aggregate++;
            // noOfExecutionsCompleted_Aggregate++;
            assertEquals(noOfExecutionCalls_Aggregate, functionServiceStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Aggregate, functionServiceStats.getFunctionExecutionsCompleted());
            FunctionStats functionStats = FunctionStats.getFunctionStats(inlineFunction.getId(), ds);
            // noOfExecutionCalls_Inline++;
            // noOfExecutionsCompleted_Inline++;
            assertEquals(noOfExecutionCalls_Inline, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Inline, functionStats.getFunctionExecutionsCompleted());
            return Boolean.TRUE;
        }
    };
    member2.invoke(checkFunctionExecutionStatsForOtherMember);
    member3.invoke(checkFunctionExecutionStatsForOtherMember);
    member4.invoke(checkFunctionExecutionStatsForOtherMember);
    SerializableCallable closeDistributedSystem = new SerializableCallable("closeDistributedSystem") {

        public Object call() throws Exception {
            if (getCache() != null && !getCache().isClosed()) {
                getCache().close();
                getCache().getDistributedSystem().disconnect();
            }
            return Boolean.TRUE;
        }
    };
    member1.invoke(closeDistributedSystem);
    member2.invoke(closeDistributedSystem);
    member3.invoke(closeDistributedSystem);
    member4.invoke(closeDistributedSystem);
}

Example 9 with Execution

use of org.apache.geode.cache.execute.Execution in project geode by apache.

the class FunctionServiceStatsDUnitTest method testClientServerPartitonedRegionFunctionExecutionStats.

/**
   * 1-client 3-Servers Function : TEST_FUNCTION2 Function : TEST_FUNCTION3 Execution of the
   * function on serverRegion with set multiple keys as the routing object and using the name of the
   * function
   * 
   * On server side, function execution calls should be equal to the no of function executions
   * completed.
   */
@Test
public void testClientServerPartitonedRegionFunctionExecutionStats() {
    createScenario();
    Function function = new TestFunction(true, TestFunction.TEST_FUNCTION2);
    registerFunctionAtServer(function);
    function = new TestFunction(true, TestFunction.TEST_FUNCTION3);
    registerFunctionAtServer(function);
    isByName = new Boolean(true);
    client.invoke(initializeStats);
    server1.invoke(initializeStats);
    server2.invoke(initializeStats);
    server3.invoke(initializeStats);
    SerializableCallable PopulateRegionAndExecuteFunctions = new SerializableCallable("PopulateRegionAndExecuteFunctions") {

        public Object call() throws Exception {
            Region region = cache.getRegion(PartitionedRegionName);
            assertNotNull(region);
            final HashSet testKeysSet = new HashSet();
            for (int i = (totalNumBuckets.intValue() * 2); i > 0; i--) {
                testKeysSet.add("execKey-" + i);
            }
            DistributedSystem.setThreadsSocketPolicy(false);
            Function function = new TestFunction(true, TestFunction.TEST_FUNCTION2);
            FunctionService.registerFunction(function);
            Execution dataSet = FunctionService.onRegion(region);
            try {
                int j = 0;
                HashSet origVals = new HashSet();
                for (Iterator i = testKeysSet.iterator(); i.hasNext(); ) {
                    Integer val = new Integer(j++);
                    origVals.add(val);
                    region.put(i.next(), val);
                }
                ResultCollector rc = dataSet.withFilter(testKeysSet).setArguments(Boolean.TRUE).execute(function.getId());
                int resultSize = ((List) rc.getResult()).size();
                resultReceived_Aggregate += resultSize;
                resultReceived_TESTFUNCTION2 += resultSize;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionCalls_TESTFUNCTION2++;
                noOfExecutionsCompleted_Aggregate++;
                noOfExecutionsCompleted_TESTFUNCTION2++;
                rc = dataSet.withFilter(testKeysSet).setArguments(testKeysSet).execute(function.getId());
                resultSize = ((List) rc.getResult()).size();
                resultReceived_Aggregate += resultSize;
                resultReceived_TESTFUNCTION2 += resultSize;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionCalls_TESTFUNCTION2++;
                noOfExecutionsCompleted_Aggregate++;
                noOfExecutionsCompleted_TESTFUNCTION2++;
                function = new TestFunction(true, TestFunction.TEST_FUNCTION3);
                FunctionService.registerFunction(function);
                rc = dataSet.withFilter(testKeysSet).setArguments(Boolean.TRUE).execute(function.getId());
                resultSize = ((List) rc.getResult()).size();
                resultReceived_Aggregate += resultSize;
                resultReceived_TESTFUNCTION3 += resultSize;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionCalls_TESTFUNCTION3++;
                noOfExecutionsCompleted_Aggregate++;
                noOfExecutionsCompleted_TESTFUNCTION3++;
            } catch (Exception e) {
                LogWriterUtils.getLogWriter().info("Exception : " + e.getMessage());
                e.printStackTrace();
                fail("Test failed after the put operation");
            }
            return Boolean.TRUE;
        }
    };
    client.invoke(PopulateRegionAndExecuteFunctions);
    SerializableCallable checkStatsOnClient = new SerializableCallable("checkStatsOnClient") {

        public Object call() throws Exception {
            // checks for the aggregate stats
            InternalDistributedSystem iDS = (InternalDistributedSystem) cache.getDistributedSystem();
            FunctionServiceStats functionServiceStats = iDS.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            assertEquals(noOfExecutionCalls_Aggregate, functionServiceStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Aggregate, functionServiceStats.getFunctionExecutionsCompleted());
            assertTrue(functionServiceStats.getResultsReceived() >= resultReceived_Aggregate);
            LogWriterUtils.getLogWriter().info("Calling FunctionStats for  TEST_FUNCTION2 :");
            FunctionStats functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION2, iDS);
            LogWriterUtils.getLogWriter().info("Called FunctionStats for  TEST_FUNCTION2 :");
            assertEquals(noOfExecutionCalls_TESTFUNCTION2, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION2, functionStats.getFunctionExecutionsCompleted());
            assertTrue(functionStats.getResultsReceived() >= resultReceived_TESTFUNCTION2);
            functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION3, iDS);
            assertEquals(noOfExecutionCalls_TESTFUNCTION3, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION3, functionStats.getFunctionExecutionsCompleted());
            assertTrue(functionStats.getResultsReceived() >= resultReceived_TESTFUNCTION3);
            return Boolean.TRUE;
        }
    };
    client.invoke(checkStatsOnClient);
    SerializableCallable checkStatsOnServer = new SerializableCallable("checkStatsOnClient") {

        public Object call() throws Exception {
            // checks for the aggregate stats
            InternalDistributedSystem iDS = (InternalDistributedSystem) cache.getDistributedSystem();
            FunctionServiceStats functionServiceStats = iDS.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            // functions are executed 3 times
            noOfExecutionCalls_Aggregate += 3;
            assertTrue(functionServiceStats.getFunctionExecutionCalls() >= noOfExecutionCalls_Aggregate);
            noOfExecutionsCompleted_Aggregate += 3;
            assertTrue(functionServiceStats.getFunctionExecutionsCompleted() >= noOfExecutionsCompleted_Aggregate);
            FunctionStats functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION2, iDS);
            // TEST_FUNCTION2 is executed twice
            noOfExecutionCalls_TESTFUNCTION2 += 2;
            assertTrue(functionStats.getFunctionExecutionCalls() >= noOfExecutionCalls_TESTFUNCTION2);
            noOfExecutionsCompleted_TESTFUNCTION2 += 2;
            assertTrue(functionStats.getFunctionExecutionsCompleted() >= noOfExecutionsCompleted_TESTFUNCTION2);
            functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION3, iDS);
            // TEST_FUNCTION3 is executed once
            noOfExecutionCalls_TESTFUNCTION3 += 1;
            assertTrue(functionStats.getFunctionExecutionCalls() >= noOfExecutionCalls_TESTFUNCTION3);
            noOfExecutionsCompleted_TESTFUNCTION3 += 1;
            assertTrue(functionStats.getFunctionExecutionsCompleted() >= noOfExecutionsCompleted_TESTFUNCTION3);
            return Boolean.TRUE;
        }
    };
    server1.invoke(checkStatsOnServer);
    server2.invoke(checkStatsOnServer);
    server3.invoke(checkStatsOnServer);
}

Example 10 with Execution

use of org.apache.geode.cache.execute.Execution in project geode by apache.

the class FunctionServiceStatsDUnitTest method testClientServerwithoutRegion.

/**
   * Execution of the function on server using the name of the function TEST_FUNCTION1
   * TEST_FUNCTION5 On client side, the no of result received should equal to the no of function
   * execution calls. On server side, function execution calls should be equal to the no of function
   * executions completed.
   */
@Test
public void testClientServerwithoutRegion() {
    createClientServerScenarionWithoutRegion();
    Function function = new TestFunction(true, TestFunction.TEST_FUNCTION1);
    registerFunctionAtServer(function);
    function = new TestFunction(true, TestFunction.TEST_FUNCTION5);
    registerFunctionAtServer(function);
    isByName = new Boolean(true);
    client.invoke(initializeStats);
    server1.invoke(initializeStats);
    server2.invoke(initializeStats);
    server3.invoke(initializeStats);
    SerializableCallable ExecuteFunction = new SerializableCallable("ExecuteFunction") {

        public Object call() throws Exception {
            DistributedSystem.setThreadsSocketPolicy(false);
            Function function = new TestFunction(true, TestFunction.TEST_FUNCTION1);
            FunctionService.registerFunction(function);
            Execution member = FunctionService.onServers(pool);
            try {
                ResultCollector rs = member.setArguments(Boolean.TRUE).execute(function.getId());
                int size = ((List) rs.getResult()).size();
                resultReceived_Aggregate += size;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionsCompleted_Aggregate++;
                resultReceived_TESTFUNCTION1 += size;
                noOfExecutionCalls_TESTFUNCTION1++;
                noOfExecutionsCompleted_TESTFUNCTION1++;
            } catch (Exception ex) {
                ex.printStackTrace();
                LogWriterUtils.getLogWriter().info("Exception : ", ex);
                fail("Test failed after the execute operation nn TRUE");
            }
            function = new TestFunction(true, TestFunction.TEST_FUNCTION5);
            FunctionService.registerFunction(function);
            try {
                final HashSet testKeysSet = new HashSet();
                for (int i = 0; i < 20; i++) {
                    testKeysSet.add("execKey-" + i);
                }
                ResultCollector rs = member.setArguments("Success").execute(function.getId());
                int size = ((List) rs.getResult()).size();
                resultReceived_Aggregate += size;
                noOfExecutionCalls_Aggregate++;
                noOfExecutionsCompleted_Aggregate++;
                resultReceived_TESTFUNCTION5 += size;
                noOfExecutionCalls_TESTFUNCTION5++;
                noOfExecutionsCompleted_TESTFUNCTION5++;
            } catch (Exception ex) {
                ex.printStackTrace();
                LogWriterUtils.getLogWriter().info("Exception : ", ex);
                fail("Test failed after the execute operationssssss");
            }
            return Boolean.TRUE;
        }
    };
    client.invoke(ExecuteFunction);
    SerializableCallable checkStatsOnClient = new SerializableCallable("checkStatsOnClient") {

        public Object call() throws Exception {
            // checks for the aggregate stats
            InternalDistributedSystem iDS = (InternalDistributedSystem) cache.getDistributedSystem();
            FunctionServiceStats functionServiceStats = iDS.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            assertEquals(noOfExecutionCalls_Aggregate, functionServiceStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_Aggregate, functionServiceStats.getFunctionExecutionsCompleted());
            assertEquals(resultReceived_Aggregate, functionServiceStats.getResultsReceived());
            FunctionStats functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION1, iDS);
            assertEquals(noOfExecutionCalls_TESTFUNCTION1, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION1, functionStats.getFunctionExecutionsCompleted());
            assertEquals(resultReceived_TESTFUNCTION1, functionStats.getResultsReceived());
            functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION5, iDS);
            assertEquals(noOfExecutionCalls_TESTFUNCTION5, functionStats.getFunctionExecutionCalls());
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION5, functionStats.getFunctionExecutionsCompleted());
            assertEquals(resultReceived_TESTFUNCTION5, functionStats.getResultsReceived());
            return Boolean.TRUE;
        }
    };
    client.invoke(checkStatsOnClient);
    SerializableCallable checkStatsOnServer = new SerializableCallable("checkStatsOnClient") {

        public Object call() throws Exception {
            // checks for the aggregate stats
            InternalDistributedSystem iDS = (InternalDistributedSystem) cache.getDistributedSystem();
            FunctionServiceStats functionServiceStats = iDS.getFunctionServiceStats();
            waitNoFunctionsRunning(functionServiceStats);
            // functions are executed 2 times
            noOfExecutionCalls_Aggregate += 2;
            assertEquals(noOfExecutionCalls_Aggregate, functionServiceStats.getFunctionExecutionCalls());
            noOfExecutionsCompleted_Aggregate += 2;
            // before giving up
            for (int i = 0; i < 10; i++) {
                try {
                    assertEquals(noOfExecutionsCompleted_Aggregate, functionServiceStats.getFunctionExecutionsCompleted());
                } catch (RuntimeException r) {
                    if (i == 9) {
                        throw r;
                    }
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt();
                        throw r;
                    }
                }
            }
            FunctionStats functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION1, iDS);
            // TEST_FUNCTION1 is executed once
            noOfExecutionCalls_TESTFUNCTION1 += 1;
            assertEquals(noOfExecutionCalls_TESTFUNCTION1, functionStats.getFunctionExecutionCalls());
            noOfExecutionsCompleted_TESTFUNCTION1 += 1;
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION1, functionStats.getFunctionExecutionsCompleted());
            functionStats = FunctionStats.getFunctionStats(TestFunction.TEST_FUNCTION5, iDS);
            // TEST_FUNCTION5 is executed once
            noOfExecutionCalls_TESTFUNCTION5 += 1;
            assertEquals(noOfExecutionCalls_TESTFUNCTION5, functionStats.getFunctionExecutionCalls());
            noOfExecutionsCompleted_TESTFUNCTION5 += 1;
            assertEquals(noOfExecutionsCompleted_TESTFUNCTION5, functionStats.getFunctionExecutionsCompleted());
            return Boolean.TRUE;
        }
    };
    server1.invoke(checkStatsOnServer);
    server2.invoke(checkStatsOnServer);
    server3.invoke(checkStatsOnServer);
}