Examples with AsyncInvocation org.apache.geode.test.dunit.AsyncInvocation used on opensource projects

Example 71 with AsyncInvocation

use of org.apache.geode.test.dunit.AsyncInvocation in project geode by apache.

the class TxnManagerMultiThreadDUnitTest method test1AllCommit.

/**
   * The following method testAllCommit test the scenario in which all four threads are committing
   * the transaction
   */
@Ignore("TODO: test is disabled")
@Test
public void test1AllCommit() throws Exception {
    VM vm0 = Host.getHost(0).getVM(0);
    AsyncInvocation asyncObj1 = vm0.invokeAsync(() -> TxnManagerMultiThreadDUnitTest.callCommitThreads());
    ThreadUtils.join(asyncObj1, 30 * 1000);
    if (asyncObj1.exceptionOccurred()) {
        Assert.fail("asyncObj1 failed", asyncObj1.getException());
    }
    vm0.invoke(() -> TxnManagerMultiThreadDUnitTest.getNumberOfRows());
}

Example 72 with AsyncInvocation

use of org.apache.geode.test.dunit.AsyncInvocation in project geode by apache.

the class DiskStoreCommandsDUnitTest method testMissingDiskStoreCommandWithColocation.

@Test
public void testMissingDiskStoreCommandWithColocation() {
    final String regionName = "testShowPersistentRecoveryFailuresRegion";
    final String childName = "childRegion";
    setUpJmxManagerOnVm0ThenConnect(null);
    final VM vm0 = Host.getHost(0).getVM(0);
    final VM vm1 = Host.getHost(0).getVM(1);
    final String vm1Name = "VM" + vm1.getPid();
    final String diskStoreName = "DiskStoreCommandsDUnitTest";
    // Default setup creates a cache in the Manager, now create a cache in VM1
    vm1.invoke(new SerializableRunnable() {

        public void run() {
            Properties localProps = new Properties();
            localProps.setProperty(NAME, vm1Name);
            getSystem(localProps);
            Cache cache = getCache();
        }
    });
    // Create a disk store and region in the Manager (VM0) and VM1 VMs
    for (final VM vm : (new VM[] { vm0, vm1 })) {
        final String vmName = "VM" + vm.getPid();
        vm.invoke(new SerializableRunnable() {

            public void run() {
                Cache cache = getCache();
                File diskStoreDirFile = new File(diskStoreName + vm.getPid());
                diskStoreDirFile.mkdirs();
                DiskStoreFactory diskStoreFactory = cache.createDiskStoreFactory();
                diskStoreFactory.setDiskDirs(new File[] { diskStoreDirFile });
                diskStoreFactory.setMaxOplogSize(1);
                diskStoreFactory.setAllowForceCompaction(true);
                diskStoreFactory.setAutoCompact(false);
                diskStoreFactory.create(regionName);
                diskStoreFactory.create(childName);
                RegionFactory regionFactory = cache.createRegionFactory();
                regionFactory.setDiskStoreName(regionName);
                regionFactory.setDiskSynchronous(true);
                regionFactory.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
                regionFactory.create(regionName);
                PartitionAttributes pa = new PartitionAttributesFactory().setColocatedWith(regionName).create();
                RegionFactory childRegionFactory = cache.createRegionFactory();
                childRegionFactory.setPartitionAttributes(pa);
                childRegionFactory.setDiskStoreName(childName);
                childRegionFactory.setDiskSynchronous(true);
                childRegionFactory.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
                childRegionFactory.create(childName);
            }
        });
    }
    // Add data to the region
    vm0.invoke(new SerializableRunnable() {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion(regionName);
            region.put("A", "a");
            region.put("B", "b");
        }
    });
    // Make sure that everything thus far is okay and there are no missing disk stores
    CommandResult cmdResult = executeCommand(CliStrings.SHOW_MISSING_DISK_STORE);
    System.out.println("command result=\n" + commandResultToString(cmdResult));
    assertEquals(Result.Status.OK, cmdResult.getStatus());
    assertTrue(cmdResult.toString(), commandResultToString(cmdResult).contains("No missing disk store found"));
    // Close the regions in the Manager (VM0) VM
    vm0.invoke(new SerializableRunnable() {

        public void run() {
            Cache cache = getCache();
            Region region = cache.getRegion(childName);
            region.close();
            region = cache.getRegion(regionName);
            region.close();
        }
    });
    // Add data to VM1 and then close the region
    vm1.invoke(new SerializableRunnable() {

        public void run() {
            Cache cache = getCache();
            Region childRegion = cache.getRegion(childName);
            PartitionedRegion parentRegion = (PartitionedRegion) (cache.getRegion(regionName));
            try {
                parentRegion.put("A", "C");
            } catch (Exception e) {
            // Ignore any exception on the put
            }
            childRegion.close();
            parentRegion.close();
        }
    });
    SerializableRunnable restartParentRegion = new SerializableRunnable("Restart parent region on") {

        public void run() {
            Cache cache = getCache();
            RegionFactory regionFactory = cache.createRegionFactory();
            regionFactory.setDiskStoreName(regionName);
            regionFactory.setDiskSynchronous(true);
            regionFactory.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
            try {
                regionFactory.create(regionName);
            } catch (Exception e) {
            // okay to ignore
            }
        }
    };
    SerializableRunnable restartChildRegion = new SerializableRunnable("Restart child region") {

        public void run() {
            Cache cache = getCache();
            PartitionAttributes pa = new PartitionAttributesFactory().setColocatedWith(regionName).create();
            RegionFactory regionFactory = cache.createRegionFactory();
            regionFactory.setPartitionAttributes(pa);
            regionFactory.setDiskStoreName(childName);
            regionFactory.setDiskSynchronous(true);
            regionFactory.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
            try {
                regionFactory.create(childName);
            } catch (Exception e) {
                // okay to ignore
                e.printStackTrace();
            }
        }
    };
    // Add the region back to the Manager (VM0) VM
    AsyncInvocation async0 = vm0.invokeAsync(restartParentRegion);
    AsyncInvocation async1 = vm1.invokeAsync(restartParentRegion);
    // Wait for the region in the Manager (VM0) to come online
    vm0.invoke(new SerializableRunnable("WaitForRegionInVm0") {

        public void run() {
            WaitCriterion waitCriterion = new WaitCriterion() {

                public boolean done() {
                    Cache cache = getCache();
                    PersistentMemberManager memberManager = ((GemFireCacheImpl) cache).getPersistentMemberManager();
                    return !memberManager.getWaitingRegions().isEmpty();
                }

                public String description() {
                    return "Waiting for another persistent member to come online";
                }
            };
            try {
                waitForCriterion(waitCriterion, 5000, 100, true);
            } catch (AssertionError ae) {
            // Ignore. waitForCriterion is expected to timeout in this test
            }
        }
    });
    // Validate that there is a missing disk store on VM1
    try {
        cmdResult = executeCommand(CliStrings.SHOW_MISSING_DISK_STORE);
        assertNotNull("Expect command result != null", cmdResult);
        assertEquals(Result.Status.OK, cmdResult.getStatus());
        String stringResult = commandResultToString(cmdResult);
        System.out.println("command result=\n" + stringResult);
        // Expect 2 result sections with header lines and 4 information lines in the first section
        assertEquals(6, countLinesInString(stringResult, false));
        assertTrue(stringContainsLine(stringResult, "Host.*Distributed Member.*Parent Region.*Missing Colocated Region"));
        assertTrue(stringContainsLine(stringResult, ".*" + regionName + ".*" + childName));
        AsyncInvocation async0b = vm0.invokeAsync(restartChildRegion);
        try {
            async0b.get(5000, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
        // Expected timeout - Region recovery is still waiting on vm1 child region and disk-store to
        // come online
        }
        cmdResult = executeCommand(CliStrings.SHOW_MISSING_DISK_STORE);
        assertNotNull("Expect command result != null", cmdResult);
        assertEquals(Result.Status.OK, cmdResult.getStatus());
        stringResult = commandResultToString(cmdResult);
        System.out.println("command result=\n" + stringResult);
        // Extract the id from the returned missing disk store
        String line = getLineFromString(stringResult, 4);
        assertFalse(line.contains("---------"));
        StringTokenizer resultTokenizer = new StringTokenizer(line);
        String id = resultTokenizer.nextToken();
        AsyncInvocation async1b = vm1.invokeAsync(restartChildRegion);
        try {
            async1b.get(5000, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
            e.printStackTrace();
        }
        cmdResult = executeCommand(CliStrings.SHOW_MISSING_DISK_STORE);
        assertNotNull("Expect command result != null", cmdResult);
        assertEquals(Result.Status.OK, cmdResult.getStatus());
        stringResult = commandResultToString(cmdResult);
        System.out.println("command result=\n" + stringResult);
    } finally {
        // Verify that the invokeAsync thread terminated
        try {
            async0.get(10000, TimeUnit.MILLISECONDS);
            async1.get(10000, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
            fail("Unexpected timeout waitiong for invokeAsync threads to terminate: " + e.getMessage());
        }
    }
    // Do our own cleanup so that the disk store directories can be removed
    super.destroyDefaultSetup();
    for (final VM vm : (new VM[] { vm0, vm1 })) {
        final String vmName = "VM" + vm.getPid();
        vm.invoke(new SerializableRunnable() {

            public void run() {
                try {
                    FileUtils.deleteDirectory((new File(diskStoreName + vm.getPid())));
                } catch (IOException iex) {
                // There's nothing else we can do
                }
            }
        });
    }
}

Example 73 with AsyncInvocation

use of org.apache.geode.test.dunit.AsyncInvocation in project geode by apache.

the class DistributedAckRegionCCEDUnitTest method testConcurrentOpWithGII.

/**
   * test for bug #45564. a create() is received by region creator and then a later destroy() is
   * received in initial image and while the version info from the destroy is recorded we keep the
   * value from the create event
   */
@Test
public void testConcurrentOpWithGII() {
    if (this.getClass() != DistributedAckRegionCCEDUnitTest.class) {
        // not really a scope-related thing
        return;
    }
    final String name = this.getUniqueName() + "-CC";
    final String key = "mykey";
    VM vm1 = Host.getHost(0).getVM(1);
    VM vm2 = Host.getHost(0).getVM(2);
    // create some destroyed entries so the GC service is populated
    SerializableCallable create = new SerializableCallable("create region") {

        public Object call() {
            RegionFactory f = getCache().createRegionFactory(getRegionAttributes());
            CCRegion = (LocalRegion) f.create(name);
            return CCRegion.getDistributionManager().getDistributionManagerId();
        }
    };
    // do conflicting update() and destroy() on the region. We want the update() to
    // be sent with a message and the destroy() to be transferred in the initial image
    // and be the value that we want to keep
    InternalDistributedMember vm1ID = (InternalDistributedMember) vm1.invoke(create);
    AsyncInvocation partialCreate = vm2.invokeAsync(new SerializableCallable("create region with stall") {

        public Object call() throws Exception {
            final GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
            RegionFactory f = cache.createRegionFactory(getRegionAttributes());
            InitialImageOperation.VMOTION_DURING_GII = true;
            // this will stall region creation at the point of asking for an initial image
            VMotionObserverHolder.setInstance(new VMotionObserver() {

                @Override
                public void vMotionBeforeCQRegistration() {
                }

                @Override
                public void vMotionBeforeRegisterInterest() {
                }

                @Override
                public void vMotionDuringGII(Set recipientSet, LocalRegion region) {
                    InitialImageOperation.VMOTION_DURING_GII = false;
                    int oldLevel = LocalRegion.setThreadInitLevelRequirement(LocalRegion.BEFORE_INITIAL_IMAGE);
                    LocalRegion ccregion = cache.getRegionByPath("/" + name);
                    try {
                        // happen
                        while (!ccregion.isDestroyed() && ccregion.getRegionEntry(key) == null) {
                            try {
                                Thread.sleep(1000);
                            } catch (InterruptedException e) {
                                return;
                            }
                        }
                    } finally {
                        LocalRegion.setThreadInitLevelRequirement(oldLevel);
                    }
                }
            });
            try {
                CCRegion = (LocalRegion) f.create(name);
                // at this point we should have received the update op and then the GII, which should
                // overwrite
                // the conflicting update op
                assertFalse("expected initial image transfer to destroy entry", CCRegion.containsKey(key));
            } finally {
                InitialImageOperation.VMOTION_DURING_GII = false;
            }
            return null;
        }
    });
    vm1.invoke(new SerializableRunnable("create conflicting events") {

        public void run() {
            // wait for the other to come on line
            long waitEnd = System.currentTimeMillis() + 45000;
            DistributionAdvisor adv = ((DistributedRegion) CCRegion).getCacheDistributionAdvisor();
            while (System.currentTimeMillis() < waitEnd && adv.adviseGeneric().isEmpty()) {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    return;
                }
            }
            if (adv.adviseGeneric().isEmpty()) {
                fail("other member never came on line");
            }
            // inhibit all messaging
            DistributedCacheOperation.LOSS_SIMULATION_RATIO = 200.0;
            try {
                CCRegion.put("mykey", "initialValue");
                CCRegion.destroy("mykey");
            } finally {
                DistributedCacheOperation.LOSS_SIMULATION_RATIO = 0.0;
            }
            // generate a fake version tag for the message
            VersionTag tag = CCRegion.getRegionEntry(key).getVersionStamp().asVersionTag();
            // create a fake member ID that will be < mine and lose a concurrency check
            NetMember nm = CCRegion.getDistributionManager().getDistributionManagerId().getNetMember();
            InternalDistributedMember mbr = null;
            try {
                mbr = new InternalDistributedMember(nm.getInetAddress().getCanonicalHostName(), nm.getPort() - 1, "fake_id", "fake_id_ustring", DistributionManager.NORMAL_DM_TYPE, null, null);
                tag.setMemberID(mbr);
            } catch (UnknownHostException e) {
                org.apache.geode.test.dunit.Assert.fail("could not create member id", e);
            }
            // generate an event to distribute that contains the fake version tag
            EntryEventImpl event = EntryEventImpl.create(CCRegion, Operation.UPDATE, key, false, mbr, true, false);
            event.setNewValue("newValue");
            event.setVersionTag(tag);
            // this should update the controller's cache with the updated value but leave this cache
            // alone
            DistributedCacheOperation op = new UpdateOperation(event, tag.getVersionTimeStamp());
            op.distribute();
            event.release();
        }
    });
    try {
        partialCreate.getResult();
    } catch (Throwable e) {
        org.apache.geode.test.dunit.Assert.fail("async invocation in vm2 failed", e);
    }
}

Example 74 with AsyncInvocation

use of org.apache.geode.test.dunit.AsyncInvocation in project geode by apache.

the class DistributedLockServiceDUnitTest method doTestSuspendLockingBehaves.

private void doTestSuspendLockingBehaves() throws Exception {
    final String dlsName = getUniqueName();
    final VM vmGrantor = Host.getHost(0).getVM(0);
    final VM vmOne = Host.getHost(0).getVM(1);
    final VM vmTwo = Host.getHost(0).getVM(2);
    final VM vmThree = Host.getHost(0).getVM(3);
    final String key1 = "key1";
    // TODO: make sure suspend thread can get other locks
    // TODO: test local (in grantor) locks and suspends also
    // define some SerializableRunnables
    final SerializableRunnable createDLS = new SerializableRunnable("Create " + dlsName) {

        public void run() {
            DistributedLockService.create(dlsName, getSystem());
            lockClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
            suspendClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
            assertFalse(isLockGrantor(dlsName).booleanValue());
        }
    };
    final SerializableRunnable suspendLocking = new SerializableRunnable("Suspend locking " + dlsName) {

        public void run() {
            suspendClientSuspendLockingBehaves.suspend();
        }
    };
    final SerializableRunnable resumeLocking = new SerializableRunnable("Resume locking " + dlsName) {

        public void run() {
            suspendClientSuspendLockingBehaves.resume();
        }
    };
    final SerializableRunnable lockKey = new SerializableRunnable("Get lock " + dlsName) {

        public void run() {
            lockClientSuspendLockingBehaves.lock();
        }
    };
    final SerializableRunnable unlockKey = new SerializableRunnable("Unlock " + dlsName) {

        public void run() {
            lockClientSuspendLockingBehaves.unlock();
        }
    };
    // create grantor
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] Create grantor " + dlsName);
    vmGrantor.invoke(new SerializableRunnable("Create grantor " + dlsName) {

        public void run() {
            DistributedLockService.create(dlsName, getSystem());
            DistributedLockService.getServiceNamed(dlsName).lock(key1, -1, -1);
            DistributedLockService.getServiceNamed(dlsName).unlock(key1);
            assertTrue(isLockGrantor(dlsName).booleanValue());
        }
    });
    // create dls in other vms
    // getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmOne");
    vmOne.invoke(createDLS);
    // getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmTwo");
    vmTwo.invoke(createDLS);
    // getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmThree");
    vmThree.invoke(createDLS);
    // get a lock
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vms for lock");
    // getLogWriter().info("[testSuspendLockingBehaves] vmOne lock");
    vmOne.invoke(lockKey);
    // getLogWriter().info("[testSuspendLockingBehaves] start vmTwoLocking");
    AsyncInvocation vmTwoLocking = vmTwo.invokeAsync(lockKey);
    // make sure vmTwo is first in line
    Wait.pause(2000);
    // getLogWriter().info("[testSuspendLockingBehaves] start vmThreeLocking");
    AsyncInvocation vmThreeLocking = vmThree.invokeAsync(lockKey);
    Wait.pause(2000);
    // make sure vmTwo and vmThree are still waiting for lock on key1
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoLocking still alive");
    Wait.pause(100);
    assertTrue(vmTwoLocking.isAlive());
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeLocking still alive");
    Wait.pause(100);
    assertTrue(vmThreeLocking.isAlive());
    // let vmTwo get key
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmTwo can get key");
    // getLogWriter().info("[testSuspendLockingBehaves] vmOne unlock");
    vmOne.invoke(unlockKey);
    ThreadUtils.join(vmTwoLocking, 10 * 1000);
    // start suspending in vmOne and vmTwo
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start suspending requests");
    // getLogWriter().info("[testSuspendLockingBehaves] start vmOneSuspending");
    AsyncInvocation vmOneSuspending = vmOne.invokeAsync(suspendLocking);
    // make sure vmOne is first in line
    Wait.pause(2000);
    // getLogWriter().info("[testSuspendLockingBehaves] start vmTwoSuspending");
    AsyncInvocation vmTwoSuspending = vmTwo.invokeAsync(suspendLocking);
    Wait.pause(2000);
    // let vmThree finish locking key
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmThree can get key");
    // getLogWriter().info("[testSuspendLockingBehaves] vmTwo unlock");
    vmTwo.invoke(unlockKey);
    ThreadUtils.join(vmThreeLocking, 10 * 1000);
    // have vmOne get back in line for locking key
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start another lock request");
    // getLogWriter().info("[testSuspendLockingBehaves] start vmOneLockingAgain");
    AsyncInvocation vmOneLockingAgain = vmOne.invokeAsync(lockKey);
    Wait.pause(2000);
    // let vmOne suspend locking
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne suspend locking");
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmOneSuspending still alive");
    Wait.pause(100);
    assertTrue(vmOneSuspending.isAlive());
    // getLogWriter().info("[testSuspendLockingBehaves] vmThree unlock");
    vmThree.invoke(unlockKey);
    ThreadUtils.join(vmOneSuspending, 10 * 1000);
    // start suspending in vmThree
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vmThree for suspending");
    // getLogWriter().info("[testSuspendLockingBehaves] start vmThreeSuspending");
    AsyncInvocation vmThreeSuspending = vmThree.invokeAsync(suspendLocking);
    Wait.pause(2000);
    // let vmTwo suspend locking
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmTwo suspend locking");
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoSuspending still alive");
    Wait.pause(100);
    assertTrue(vmTwoSuspending.isAlive());
    // getLogWriter().info("[testSuspendLockingBehaves] vmOne resumes locking");
    vmOne.invoke(resumeLocking);
    ThreadUtils.join(vmTwoSuspending, 10 * 1000);
    // let vmOne get that lock
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne get that lock");
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmOneLockingAgain still alive");
    Wait.pause(100);
    assertTrue(vmOneLockingAgain.isAlive());
    // getLogWriter().info("[testSuspendLockingBehaves] vmTwo resumes locking");
    vmTwo.invoke(resumeLocking);
    ThreadUtils.join(vmOneLockingAgain, 10 * 1000);
    // let vmThree suspend locking
    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmThree suspend locking");
    // getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeSuspending still alive");
    Wait.pause(100);
    assertTrue(vmThreeSuspending.isAlive());
    // getLogWriter().info("[testSuspendLockingBehaves] vmOne unlocks again");
    vmOne.invoke(unlockKey);
    ThreadUtils.join(vmThreeSuspending, 10 * 1000);
    // done
    // getLogWriter().info("[testSuspendLockingBehaves] vmThree resumes locking");
    vmThree.invoke(resumeLocking);
}

Example 75 with AsyncInvocation

use of org.apache.geode.test.dunit.AsyncInvocation in project geode by apache.

the class DistributedLockServiceDUnitTest method testLockQuery.

@Test
public void testLockQuery() throws Exception {
    final String dlsName = getUniqueName();
    final VM vmGrantor = Host.getHost(0).getVM(0);
    final VM vm1 = Host.getHost(0).getVM(1);
    final VM vm2 = Host.getHost(0).getVM(2);
    final String key1 = "key1";
    // vmGrantor creates grantor
    vmGrantor.invoke(new SerializableRunnable() {

        public void run() {
            LogWriterUtils.getLogWriter().info("[testLockQuery] vmGrantor creates grantor");
            connectDistributedSystem();
            DLockService dls = (DLockService) DistributedLockService.create(dlsName, getSystem());
            assertTrue(dls.lock(key1, -1, -1));
            assertTrue(dls.isLockGrantor());
            dls.unlock(key1);
            dls.freeResources(key1);
        }
    });
    AsyncInvocation whileVM1Locks = null;
    try {
        // vm1 locks key1
        whileVM1Locks = vm1.invokeAsync(new SerializableRunnable() {

            public void run() {
                LogWriterUtils.getLogWriter().info("[testLockQuery] vm1 locks key1");
                connectDistributedSystem();
                DLockService dls = (DLockService) DistributedLockService.create(dlsName, getSystem());
                assertTrue(dls.lock(key1, -1, -1));
                assertFalse(dls.isLockGrantor());
                try {
                    synchronized (testLockQuery_whileVM1Locks) {
                        testLockQuery_whileVM1Locks.set(true);
                        testLockQuery_whileVM1Locks.notifyAll();
                        long maxWait = 10000;
                        StopWatch timer = new StopWatch(true);
                        while (testLockQuery_whileVM1Locks.get()) {
                            // while true
                            long timeLeft = maxWait - timer.elapsedTimeMillis();
                            if (timeLeft > 0) {
                                testLockQuery_whileVM1Locks.wait(timeLeft);
                            } else {
                                fail("Test attempted to wait too long");
                            }
                        }
                    }
                } catch (InterruptedException e) {
                    org.apache.geode.test.dunit.Assert.fail(e.getMessage(), e);
                }
                LogWriterUtils.getLogWriter().info("[testLockQuery] vm1 unlocks key1");
                dls.unlock(key1);
                dls.freeResources(key1);
            }
        });
        // wait for vm1 to set testLockQuery_whileVM1Locks
        // get DistributedMember for vm1
        final DistributedMember vm1Member = (DistributedMember) vm1.invoke(new SerializableCallable() {

            public Object call() throws Exception {
                LogWriterUtils.getLogWriter().info("[testLockQuery] vm1 waits for locking thread");
                synchronized (testLockQuery_whileVM1Locks) {
                    long maxWait = 10000;
                    StopWatch timer = new StopWatch(true);
                    while (!testLockQuery_whileVM1Locks.get()) {
                        // while false
                        long timeLeft = maxWait - timer.elapsedTimeMillis();
                        if (timeLeft > 0) {
                            testLockQuery_whileVM1Locks.wait(timeLeft);
                        } else {
                            fail("Test attempted to wait too long");
                        }
                    }
                }
                return getSystem().getDistributedMember();
            }
        });
        assertNotNull(vm1Member);
        // vmGrantor tests positive local dlock query
        vmGrantor.invoke(new SerializableRunnable() {

            public void run() {
                LogWriterUtils.getLogWriter().info("[testLockQuery] vmGrantor tests local query");
                DLockService dls = (DLockService) DistributedLockService.getServiceNamed(dlsName);
                DLockRemoteToken result = dls.queryLock(key1);
                assertNotNull(result);
                assertEquals(key1, result.getName());
                assertTrue(result.getLeaseId() != -1);
                assertEquals(Long.MAX_VALUE, result.getLeaseExpireTime());
                RemoteThread lesseeThread = result.getLesseeThread();
                assertNotNull(lesseeThread);
                assertEquals(vm1Member, lesseeThread.getDistributedMember());
                assertEquals(vm1Member, result.getLessee());
            // nothing to test for on threadId unless we serialize info from vm1
            }
        });
        // vm2 tests positive remote dlock query
        vm2.invoke(new SerializableRunnable() {

            public void run() {
                LogWriterUtils.getLogWriter().info("[testLockQuery] vm2 tests remote query");
                connectDistributedSystem();
                DLockService dls = (DLockService) DistributedLockService.create(dlsName, getSystem());
                DLockRemoteToken result = dls.queryLock(key1);
                assertNotNull(result);
                assertEquals(key1, result.getName());
                assertTrue(result.getLeaseId() != -1);
                assertEquals(Long.MAX_VALUE, result.getLeaseExpireTime());
                RemoteThread lesseeThread = result.getLesseeThread();
                assertNotNull(lesseeThread);
                assertEquals(vm1Member, lesseeThread.getDistributedMember());
                assertEquals(vm1Member, result.getLessee());
            // nothing to test for on threadId unless we serialize info from vm1
            }
        });
    } finally {
        // guarantee that testLockQuery_whileVM1Locks is notfied!
        // vm1 sets and notifies testLockQuery_whileVM1Locks to release lock
        vm1.invoke(new SerializableRunnable() {

            public void run() {
                LogWriterUtils.getLogWriter().info("[testLockQuery] vm1 notifies/releases key1");
                synchronized (testLockQuery_whileVM1Locks) {
                    testLockQuery_whileVM1Locks.set(false);
                    testLockQuery_whileVM1Locks.notifyAll();
                }
            }
        });
        ThreadUtils.join(whileVM1Locks, 10 * 1000);
        if (whileVM1Locks.exceptionOccurred()) {
            org.apache.geode.test.dunit.Assert.fail("Test failed", whileVM1Locks.getException());
        }
    }
    // vmGrantor tests negative local dlock query
    vmGrantor.invoke(new SerializableRunnable() {

        public void run() {
            LogWriterUtils.getLogWriter().info("[testLockQuery] vmGrantor tests negative query");
            DLockService dls = (DLockService) DistributedLockService.getServiceNamed(dlsName);
            DLockRemoteToken result = dls.queryLock(key1);
            assertNotNull(result);
            assertEquals(key1, result.getName());
            assertEquals(-1, result.getLeaseId());
            assertEquals(0, result.getLeaseExpireTime());
            assertNull(result.getLesseeThread());
            assertNull(result.getLessee());
        }
    });
    // vm2 tests negative remote dlock query
    vm2.invoke(new SerializableRunnable() {

        public void run() {
            LogWriterUtils.getLogWriter().info("[testLockQuery] vm2 tests negative query");
            DLockService dls = (DLockService) DistributedLockService.getServiceNamed(dlsName);
            DLockRemoteToken result = dls.queryLock(key1);
            assertNotNull(result);
            assertEquals(key1, result.getName());
            assertEquals(-1, result.getLeaseId());
            assertEquals(0, result.getLeaseExpireTime());
            assertNull(result.getLesseeThread());
            assertNull(result.getLessee());
        }
    });
}