Examples with LocalLogWriter org.apache.geode.internal.logging.LocalLogWriter used on opensource projects

Example 6 with LocalLogWriter

use of org.apache.geode.internal.logging.LocalLogWriter in project geode by apache.

the class LocatorDUnitTest method testLocatorBecomesCoordinator.

/**
   * Tests starting one locator in a remote VM and having multiple members of the distributed system
   * join it. This ensures that members start up okay, and that handling of a stopped locator is
   * correct. It then restarts the locator to demonstrate that it can connect to and function as the
   * group coordinator
   */
@Test
public void testLocatorBecomesCoordinator() throws Exception {
    disconnectAllFromDS();
    final String expected = "java.net.ConnectException";
    final String addExpected = "<ExpectedException action=add>" + expected + "</ExpectedException>";
    final String removeExpected = "<ExpectedException action=remove>" + expected + "</ExpectedException>";
    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    final int port = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
    DistributedTestUtils.deleteLocatorStateFile(port1);
    final String locators = NetworkUtils.getServerHostName(host) + "[" + port + "]";
    vm0.invoke(getUniqueName() + 1, () -> startSBLocator(port));
    try {
        final Properties props = new Properties();
        props.setProperty(LOCATORS, locators);
        props.setProperty(ENABLE_NETWORK_PARTITION_DETECTION, "true");
        props.put(ENABLE_CLUSTER_CONFIGURATION, "false");
        addDSProps(props);
        vm1.invoke(() -> {
            DistributedSystem sys = getSystem(props);
            sys.getLogWriter().info(addExpected);
        });
        vm2.invoke(() -> {
            DistributedSystem sys = getSystem(props);
            sys.getLogWriter().info(addExpected);
        });
        system = (InternalDistributedSystem) getSystem(props);
        final DistributedMember coord = MembershipManagerHelper.getCoordinator(system);
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("coordinator before termination of locator is " + coord);
        vm0.invoke(() -> stopLocator());
        // now ensure that one of the remaining members became the coordinator
        Awaitility.waitAtMost(15000, TimeUnit.MILLISECONDS).pollInterval(200, TimeUnit.MILLISECONDS).until(() -> !coord.equals(MembershipManagerHelper.getCoordinator(system)));
        DistributedMember newCoord = MembershipManagerHelper.getCoordinator(system);
        LogWriterUtils.getLogWriter().info("coordinator after shutdown of locator was " + newCoord);
        if (newCoord == null || coord.equals(newCoord)) {
            fail("another member should have become coordinator after the locator was stopped: " + newCoord);
        }
        // restart the locator to demonstrate reconnection & make disconnects faster
        // it should also regain the role of coordinator, so we check to make sure
        // that the coordinator has changed
        vm0.invoke(getUniqueName() + "2", () -> startSBLocator(port));
        final DistributedMember tempCoord = newCoord;
        Awaitility.waitAtMost(5000, TimeUnit.MILLISECONDS).pollInterval(200, TimeUnit.MILLISECONDS).until(() -> !tempCoord.equals(MembershipManagerHelper.getCoordinator(system)));
        system.disconnect();
        LogWriter bgexecLogger = new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
        bgexecLogger.info(removeExpected);
        SerializableRunnable disconnect = new SerializableRunnable("Disconnect from " + locators) {

            public void run() {
            }
        };
        vm1.invoke(() -> {
            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
            if (sys != null && sys.isConnected()) {
                sys.disconnect();
            }
            // connectExceptions occur during disconnect, so we need the
            // expectedexception hint to be in effect until this point
            LogWriter bLogger = new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
            bLogger.info(removeExpected);
        });
        vm2.invoke(() -> {
            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
            if (sys != null && sys.isConnected()) {
                sys.disconnect();
            }
            // connectExceptions occur during disconnect, so we need the
            // expectedexception hint to be in effect until this point
            LogWriter bLogger = new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
            bLogger.info(removeExpected);
        });
        vm0.invoke(() -> stopLocator());
    } finally {
        vm0.invoke(() -> stopLocator());
    }
}

Example 7 with LocalLogWriter

use of org.apache.geode.internal.logging.LocalLogWriter in project geode by apache.

the class LocatorDUnitTest method testLeadAndCoordFailure.

/**
   * test lead member and coordinator failure with network partition detection enabled. It would be
   * nice for this test to have more than two "server" vms, to demonstrate that they all exit when
   * the leader and potential- coordinator both disappear in the loss-correlation-window, but there
   * are only four vms available for dunit testing.
   * <p>
   * So, we start two locators with admin distributed systems, then start two regular distributed
   * members.
   * <p>
   * We kill the second locator (which is not the view coordinator) and then kill the non-lead
   * member. That should be okay - the lead and remaining locator continue to run.
   * <p>
   * We then kill the lead member and demonstrate that the original locator (which is now the sole
   * remaining member) shuts itself down.
   */
@Test
public void testLeadAndCoordFailure() throws Exception {
    IgnoredException.addIgnoredException("Possible loss of quorum due");
    disconnectAllFromDS();
    Host host = Host.getHost(0);
    VM vm1 = host.getVM(1);
    VM vm2 = host.getVM(2);
    VM locvm = host.getVM(3);
    Locator locator = null;
    int[] ports = AvailablePortHelper.getRandomAvailableTCPPorts(2);
    final int port1 = ports[0];
    this.port1 = port1;
    final int port2 = ports[1];
    DistributedTestUtils.deleteLocatorStateFile(port1, port2);
    final String host0 = NetworkUtils.getServerHostName(host);
    final String locators = host0 + "[" + port1 + "]," + host0 + "[" + port2 + "]";
    final Properties properties = new Properties();
    properties.put(MCAST_PORT, "0");
    properties.put(LOCATORS, locators);
    properties.put(ENABLE_NETWORK_PARTITION_DETECTION, "true");
    properties.put(DISABLE_AUTO_RECONNECT, "true");
    properties.put(MEMBER_TIMEOUT, "2000");
    properties.put(LOG_LEVEL, LogWriterUtils.getDUnitLogLevel());
    // properties.put("log-level", "fine");
    properties.put(ENABLE_CLUSTER_CONFIGURATION, "false");
    addDSProps(properties);
    try {
        final String uname = getUniqueName();
        File logFile = new File("");
        locator = Locator.startLocatorAndDS(port1, logFile, properties);
        final DistributedSystem sys = locator.getDistributedSystem();
        sys.getLogWriter().info("<ExpectedException action=add>java.net.ConnectException</ExpectedException>");
        MembershipManagerHelper.inhibitForcedDisconnectLogging(true);
        locvm.invoke(new SerializableRunnable() {

            public void run() {
                File lf = new File("");
                try {
                    Locator.startLocatorAndDS(port2, lf, properties);
                } catch (IOException ios) {
                    org.apache.geode.test.dunit.Assert.fail("Unable to start locator2", ios);
                }
            }
        });
        Object[] connectArgs = new Object[] { properties };
        SerializableRunnable crashLocator = new SerializableRunnable("Crash locator") {

            public void run() {
                Locator loc = Locator.getLocators().iterator().next();
                DistributedSystem msys = loc.getDistributedSystem();
                MembershipManagerHelper.crashDistributedSystem(msys);
                loc.stop();
            }
        };
        assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
        // properties.put("log-level", getDUnitLogLevel());
        DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(), "getDistributedMember", connectArgs);
        vm2.invoke(this.getClass(), "getDistributedMember", connectArgs);
        assertLeadMember(mem1, sys, 5000);
        assertEquals(sys.getDistributedMember(), MembershipManagerHelper.getCoordinator(sys));
        // crash the second vm and the locator. Should be okay
        DistributedTestUtils.crashDistributedSystem(vm2);
        locvm.invoke(crashLocator);
        assertTrue("Distributed system should not have disconnected", vm1.invoke(() -> LocatorDUnitTest.isSystemConnected()));
        // ensure quorumLost is properly invoked
        DistributionManager dm = (DistributionManager) ((InternalDistributedSystem) sys).getDistributionManager();
        MyMembershipListener listener = new MyMembershipListener();
        dm.addMembershipListener(listener);
        // ensure there is an unordered reader thread for the member
        new HighPriorityAckedMessage().send(Collections.singleton(mem1), false);
        // disconnect the first vm and demonstrate that the third vm and the
        // locator notice the failure and exit
        DistributedTestUtils.crashDistributedSystem(vm1);
        /*
       * This vm is watching vm1, which is watching vm2 which is watching locvm. It will take 3 * (3
       * * member-timeout) milliseconds to detect the full failure and eject the lost members from
       * the view.
       */
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("waiting for my distributed system to disconnect due to partition detection");
        Awaitility.waitAtMost(24000, TimeUnit.MILLISECONDS).pollInterval(200, TimeUnit.MILLISECONDS).until(() -> {
            return !sys.isConnected();
        });
        if (sys.isConnected()) {
            fail("Distributed system did not disconnect as expected - network partition detection is broken");
        }
        // quorumLost should be invoked if we get a ForcedDisconnect in this situation
        assertTrue("expected quorumLost to be invoked", listener.quorumLostInvoked);
        assertTrue("expected suspect processing initiated by TCPConduit", listener.suspectReasons.contains(Connection.INITIATING_SUSPECT_PROCESSING));
    } finally {
        if (locator != null) {
            locator.stop();
        }
        LogWriter bLogger = new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
        bLogger.info("<ExpectedException action=remove>service failure</ExpectedException>");
        bLogger.info("<ExpectedException action=remove>java.net.ConnectException</ExpectedException>");
        bLogger.info("<ExpectedException action=remove>org.apache.geode.ForcedDisconnectException</ExpectedException>");
        disconnectAllFromDS();
    }
}

Example 8 with LocalLogWriter

use of org.apache.geode.internal.logging.LocalLogWriter in project geode by apache.

the class OpExecutorImplJUnitTest method setUp.

@Before
public void setUp() {
    this.logger = new LocalLogWriter(InternalLogWriter.FINEST_LEVEL, System.out);
    this.endpointManager = new DummyEndpointManager();
    this.queueManager = new DummyQueueManager();
    this.manager = new DummyManager();
    riTracker = new RegisterInterestTracker();
    cancelCriterion = new CancelCriterion() {

        @Override
        public String cancelInProgress() {
            return null;
        }

        @Override
        public RuntimeException generateCancelledException(Throwable e) {
            return null;
        }
    };
}

Example 9 with LocalLogWriter

use of org.apache.geode.internal.logging.LocalLogWriter in project geode by apache.

the class UniversalMembershipListenerAdapterDUnitTest method doTestServerEventsInPeerSystem.

private void doTestServerEventsInPeerSystem() throws Exception {
    final boolean[] firedSystem = new boolean[3];
    final DistributedMember[] memberSystem = new DistributedMember[3];
    final String[] memberIdSystem = new String[3];
    final boolean[] isClientSystem = new boolean[3];
    final boolean[] firedAdapter = new boolean[3];
    final DistributedMember[] memberAdapter = new DistributedMember[3];
    final String[] memberIdAdapter = new String[3];
    final boolean[] isClientAdapter = new boolean[3];
    final boolean[] firedBridge = new boolean[3];
    final DistributedMember[] memberBridge = new DistributedMember[3];
    final String[] memberIdBridge = new String[3];
    final boolean[] isClientBridge = new boolean[3];
    final boolean[] firedSystemDuplicate = new boolean[3];
    final boolean[] firedAdapterDuplicate = new boolean[3];
    final boolean[] firedBridgeDuplicate = new boolean[3];
    MembershipListener systemListener = new MembershipListener() {

        public synchronized void memberJoined(MembershipEvent event) {
            firedSystemDuplicate[JOINED] = firedSystem[JOINED];
            firedSystem[JOINED] = true;
            memberSystem[JOINED] = event.getDistributedMember();
            memberIdSystem[JOINED] = event.getMemberId();
            notify();
        }

        public synchronized void memberLeft(MembershipEvent event) {
            firedSystemDuplicate[LEFT] = firedSystem[LEFT];
            firedSystem[LEFT] = true;
            memberSystem[LEFT] = event.getDistributedMember();
            memberIdSystem[LEFT] = event.getMemberId();
            notify();
        }

        public synchronized void memberCrashed(MembershipEvent event) {
            firedSystemDuplicate[CRASHED] = firedSystem[CRASHED];
            firedSystem[CRASHED] = true;
            memberSystem[CRASHED] = event.getDistributedMember();
            memberIdSystem[CRASHED] = event.getMemberId();
            notify();
        }
    };
    UniversalMembershipListenerAdapter adapter = new UniversalMembershipListenerAdapter() {

        @Override
        public synchronized void memberJoined(MembershipEvent event) {
            getLogWriter().info("[testServerEventsInSystemClient] memberJoined >" + event.getMemberId() + "<");
            firedAdapterDuplicate[JOINED] = firedAdapter[JOINED];
            firedAdapter[JOINED] = true;
            memberAdapter[JOINED] = event.getDistributedMember();
            memberIdAdapter[JOINED] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[JOINED] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }

        @Override
        public synchronized void memberLeft(MembershipEvent event) {
            getLogWriter().info("[testServerEventsInSystemClient] memberLeft >" + event.getMemberId() + "<");
            firedAdapterDuplicate[LEFT] = firedAdapter[LEFT];
            firedAdapter[LEFT] = true;
            memberAdapter[LEFT] = event.getDistributedMember();
            memberIdAdapter[LEFT] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[LEFT] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }

        @Override
        public synchronized void memberCrashed(MembershipEvent event) {
            getLogWriter().info("[testServerEventsInSystemClient] memberCrashed >" + event.getMemberId() + "<");
            firedAdapterDuplicate[CRASHED] = firedAdapter[CRASHED];
            firedAdapter[CRASHED] = true;
            memberAdapter[CRASHED] = event.getDistributedMember();
            memberIdAdapter[CRASHED] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[CRASHED] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }
    };
    ClientMembershipListener bridgeListener = new ClientMembershipListener() {

        public synchronized void memberJoined(ClientMembershipEvent event) {
            firedBridgeDuplicate[JOINED] = firedBridge[JOINED];
            firedBridge[JOINED] = true;
            memberBridge[JOINED] = event.getMember();
            memberIdBridge[JOINED] = event.getMemberId();
            isClientBridge[JOINED] = event.isClient();
            notify();
        }

        public synchronized void memberLeft(ClientMembershipEvent event) {
            firedBridgeDuplicate[LEFT] = firedBridge[LEFT];
            firedBridge[LEFT] = true;
            memberBridge[LEFT] = event.getMember();
            memberIdBridge[LEFT] = event.getMemberId();
            isClientBridge[LEFT] = event.isClient();
            notify();
        }

        public synchronized void memberCrashed(ClientMembershipEvent event) {
            firedBridgeDuplicate[CRASHED] = firedBridge[CRASHED];
            firedBridge[CRASHED] = true;
            memberBridge[CRASHED] = event.getMember();
            memberIdBridge[CRASHED] = event.getMemberId();
            isClientBridge[CRASHED] = event.isClient();
            notify();
        }
    };
    final Host host = Host.getHost(0);
    final VM vm0 = host.getVM(0);
    final String name = this.getUniqueName();
    final int[] ports = new int[] { AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET) };
    assertTrue(ports[0] != 0);
    // create BridgeServer in controller vm...
    System.out.println("[doTestSystemClientEventsInServer] Create BridgeServer");
    getSystem();
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(Scope.LOCAL);
    Region region = createRegion(name, factory.create());
    assertNotNull(region);
    assertNotNull(getRootRegion().getSubregion(name));
    ports[0] = startBridgeServer(0);
    assertTrue(ports[0] != 0);
    final DistributedMember serverMember = getMemberId();
    final String serverMemberId = serverMember.getId();
    final Properties serverProperties = getSystem().getProperties();
    serverProperties.remove(CLUSTER_SSL_ENABLED);
    serverProperties.remove(CLUSTER_SSL_CIPHERS);
    serverProperties.remove(CLUSTER_SSL_PROTOCOLS);
    serverProperties.remove(CLUSTER_SSL_REQUIRE_AUTHENTICATION);
    System.out.println("[testServerEventsInPeerSystem] ports[0]=" + ports[0]);
    System.out.println("[testServerEventsInPeerSystem] serverMemberId=" + serverMemberId);
    System.out.println("[testServerEventsInPeerSystem] serverMember=" + serverMember);
    GemFireCacheImpl cache = GemFireCacheImpl.getExisting();
    assertNotNull(cache);
    ManagementService service = ManagementService.getExistingManagementService(cache);
    // register the system listener
    service.addMembershipListener(systemListener);
    // register the universal adapter.
    adapter.registerMembershipListener(service);
    // create BridgeServer in vm0...
    SerializableCallable createPeerCache = new SerializableCallable("Create Peer Cache") {

        @Override
        public Object call() {
            System.out.println("[testServerEventsInPeerSystem] Create Peer cache");
            getSystem(serverProperties);
            AttributesFactory factory = new AttributesFactory();
            factory.setScope(Scope.LOCAL);
            Region region = createRegion(name, factory.create());
            assertNotNull(region);
            assertNotNull(getRootRegion().getSubregion(name));
            return basicGetSystem().getDistributedMember();
        }
    };
    DistributedMember peerMember = (DistributedMember) vm0.invoke(createPeerCache);
    String peerMemberId = peerMember.getId();
    System.out.println("[testServerEventsInPeerSystem] peerMemberId=" + peerMemberId);
    System.out.println("[testServerEventsInPeerSystem] peerMember=" + peerMember);
    synchronized (systemListener) {
        if (!firedSystem[JOINED]) {
            systemListener.wait(ASYNC_EVENT_WAIT_MILLIS);
        }
    }
    synchronized (adapter) {
        if (!firedAdapter[JOINED]) {
            adapter.wait(ASYNC_EVENT_WAIT_MILLIS);
        }
    }
    System.out.println("[testServerEventsInPeerSystem] assert server detected peer join");
    assertArrayFalse(firedSystemDuplicate);
    assertTrue(firedSystem[JOINED]);
    assertEquals(peerMember, memberSystem[JOINED]);
    assertEquals(peerMemberId, memberIdSystem[JOINED]);
    assertFalse(isClientSystem[JOINED]);
    assertFalse(firedSystem[LEFT]);
    assertNull(memberSystem[LEFT]);
    assertNull(memberIdSystem[LEFT]);
    assertFalse(isClientSystem[LEFT]);
    assertFalse(firedSystem[CRASHED]);
    assertNull(memberSystem[CRASHED]);
    assertNull(memberIdSystem[CRASHED]);
    assertFalse(isClientSystem[CRASHED]);
    resetArraysForTesting(firedSystem, memberSystem, memberIdSystem, isClientSystem);
    assertTrue(firedAdapter[JOINED]);
    assertNotNull(memberAdapter[JOINED]);
    assertNotNull(memberIdAdapter[JOINED]);
    assertEquals(peerMember, memberAdapter[JOINED]);
    assertEquals(peerMemberId, memberIdAdapter[JOINED]);
    assertFalse(isClientAdapter[JOINED]);
    assertFalse(firedAdapter[LEFT]);
    assertNull(memberAdapter[LEFT]);
    assertNull(memberIdAdapter[LEFT]);
    assertFalse(isClientAdapter[LEFT]);
    assertFalse(firedAdapter[CRASHED]);
    assertNull(memberAdapter[CRASHED]);
    assertNull(memberIdAdapter[CRASHED]);
    assertFalse(isClientAdapter[CRASHED]);
    resetArraysForTesting(firedAdapter, memberAdapter, memberIdAdapter, isClientAdapter);
    LogWriter bgexecLogger = new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
    bgexecLogger.info("<ExpectedException action=add>" + "java.io.IOException" + "</ExpectedException>");
    final IgnoredException ex = IgnoredException.addIgnoredException(ServerConnectivityException.class.getName());
    try {
        vm0.invoke(new SerializableRunnable("Disconnect Peer server") {

            public void run() {
                System.out.println("[testServerEventsInPeerSystem] disconnect peer server");
                closeCache();
                disconnectFromDS();
            }
        });
        synchronized (systemListener) {
            if (!firedSystem[LEFT]) {
                systemListener.wait(ASYNC_EVENT_WAIT_MILLIS);
            }
        }
        synchronized (adapter) {
            if (!firedAdapter[LEFT]) {
                adapter.wait(ASYNC_EVENT_WAIT_MILLIS);
            }
        }
    } finally {
        bgexecLogger.info("<ExpectedException action=remove>" + "java.io.IOException" + "</ExpectedException>");
        ex.remove();
    }
    System.out.println("[testServerEventsInPeerSystem] assert server detected peer crashed");
    assertArrayFalse(firedSystemDuplicate);
    assertArrayFalse(firedAdapterDuplicate);
    assertFalse(firedSystem[JOINED]);
    assertNull(memberSystem[JOINED]);
    assertNull(memberIdSystem[JOINED]);
    assertFalse(isClientSystem[JOINED]);
    assertTrue(firedSystem[LEFT]);
    assertEquals(peerMember, memberSystem[LEFT]);
    assertEquals(peerMemberId, memberIdSystem[LEFT]);
    assertFalse(isClientSystem[LEFT]);
    assertFalse(firedSystem[CRASHED]);
    assertNull(memberSystem[CRASHED]);
    assertNull(memberIdSystem[CRASHED]);
    assertFalse(isClientSystem[CRASHED]);
    resetArraysForTesting(firedSystem, memberSystem, memberIdSystem, isClientSystem);
    assertFalse("this intermittently fails", firedAdapter[JOINED]);
    assertNull(memberIdAdapter[JOINED]);
    assertFalse(isClientAdapter[JOINED]);
    // LEFT fired by System listener
    assertTrue(firedAdapter[LEFT]);
    assertEquals(peerMember, memberAdapter[LEFT]);
    assertEquals(peerMemberId, memberIdAdapter[LEFT]);
    assertFalse(isClientAdapter[LEFT]);
    // There won't be an adapter crashed event because since the two VMs
    // are in the same distributed system, and the server's real member
    // id is used now. In this case, two events are sent - one from
    // jgroups (memberDeparted), and one from the server (a memberCrshed).
    // The memberCrashed event is deemed a duplicate and not sent - see
    // UniversalMembershipListenerAdapter.MembershipListener.isDuplicate
    assertFalse(firedAdapter[CRASHED]);
    assertNull(memberAdapter[CRASHED]);
    assertNull(memberIdAdapter[CRASHED]);
    assertFalse(isClientAdapter[CRASHED]);
    resetArraysForTesting(firedAdapter, memberAdapter, memberIdAdapter, isClientAdapter);
}

Example 10 with LocalLogWriter

use of org.apache.geode.internal.logging.LocalLogWriter in project geode by apache.

the class UniversalMembershipListenerAdapterDUnitTest method doTestServerEventsInLonerClient.

private void doTestServerEventsInLonerClient() throws Exception {
    final boolean[] firedAdapter = new boolean[3];
    final DistributedMember[] memberAdapter = new DistributedMember[3];
    final String[] memberIdAdapter = new String[3];
    final boolean[] isClientAdapter = new boolean[3];
    final boolean[] firedBridge = new boolean[3];
    final DistributedMember[] memberBridge = new DistributedMember[3];
    final String[] memberIdBridge = new String[3];
    final boolean[] isClientBridge = new boolean[3];
    final boolean[] firedAdapterDuplicate = new boolean[3];
    final boolean[] firedBridgeDuplicate = new boolean[3];
    UniversalMembershipListenerAdapter adapter = new UniversalMembershipListenerAdapter() {

        @Override
        public synchronized void memberJoined(MembershipEvent event) {
            System.out.println("[testServerEventsInLonerClient] memberJoined >" + event.getMemberId() + "<");
            firedAdapterDuplicate[JOINED] = firedAdapter[JOINED];
            firedAdapter[JOINED] = true;
            memberAdapter[JOINED] = event.getDistributedMember();
            memberIdAdapter[JOINED] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[JOINED] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }

        @Override
        public synchronized void memberLeft(MembershipEvent event) {
            System.out.println("[testServerEventsInLonerClient] memberLeft >" + event.getMemberId() + "<");
            firedAdapterDuplicate[LEFT] = firedAdapter[LEFT];
            firedAdapter[LEFT] = true;
            memberAdapter[LEFT] = event.getDistributedMember();
            memberIdAdapter[LEFT] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[LEFT] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }

        @Override
        public synchronized void memberCrashed(MembershipEvent event) {
            System.out.println("[testServerEventsInLonerClient] memberCrashed >" + event.getMemberId() + "<");
            firedAdapterDuplicate[CRASHED] = firedAdapter[CRASHED];
            firedAdapter[CRASHED] = true;
            memberAdapter[CRASHED] = event.getDistributedMember();
            memberIdAdapter[CRASHED] = event.getMemberId();
            if (event instanceof UniversalMembershipListenerAdapter.AdaptedMembershipEvent) {
                isClientAdapter[CRASHED] = ((UniversalMembershipListenerAdapter.AdaptedMembershipEvent) event).isClient();
            }
            notify();
        }
    };
    ClientMembershipListener bridgeListener = new ClientMembershipListener() {

        public synchronized void memberJoined(ClientMembershipEvent event) {
            firedBridgeDuplicate[JOINED] = firedBridge[JOINED];
            firedBridge[JOINED] = true;
            memberBridge[JOINED] = event.getMember();
            memberIdBridge[JOINED] = event.getMemberId();
            isClientBridge[JOINED] = event.isClient();
            notify();
        }

        public synchronized void memberLeft(ClientMembershipEvent event) {
            firedBridgeDuplicate[LEFT] = firedBridge[LEFT];
            firedBridge[LEFT] = true;
            memberBridge[LEFT] = event.getMember();
            memberIdBridge[LEFT] = event.getMemberId();
            isClientBridge[LEFT] = event.isClient();
            notify();
        }

        public synchronized void memberCrashed(ClientMembershipEvent event) {
            firedBridgeDuplicate[CRASHED] = firedBridge[CRASHED];
            firedBridge[CRASHED] = true;
            memberBridge[CRASHED] = event.getMember();
            memberIdBridge[CRASHED] = event.getMemberId();
            isClientBridge[CRASHED] = event.isClient();
            notify();
        }
    };
    final Host host = Host.getHost(0);
    final VM vm0 = host.getVM(0);
    final String name = this.getUniqueName();
    final int[] ports = new int[] { AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET) };
    assertTrue(ports[0] != 0);
    System.out.println("[testServerEventsInLonerClient] create loner bridge client");
    Properties config = new Properties();
    config.put(MCAST_PORT, "0");
    config.put(LOCATORS, "");
    // config.put(LOG_LEVEL, "fine");
    config.setProperty(ENABLE_NETWORK_PARTITION_DETECTION, "false");
    getSystem(config);
    System.out.println("[testServerEventsInLonerClient] create system bridge client");
    getSystem();
    // register the bridge listener
    ClientMembership.registerClientMembershipListener(bridgeListener);
    // adapter should've self-registered w/ BridgeMembership
    // String clientMemberId = getMemberId();
    // DistributedMember clientMember = getDistributedMember();
    // create BridgeServer in vm0...
    SerializableCallable createBridgeServer = new SerializableCallable("Create BridgeServer") {

        @Override
        public Object call() {
            System.out.println("[testServerEventsInLonerClient] Create BridgeServer");
            getSystem();
            AttributesFactory factory = new AttributesFactory();
            factory.setScope(Scope.LOCAL);
            Region region = createRegion(name, factory.create());
            assertNotNull(region);
            assertNotNull(getRootRegion().getSubregion(name));
            try {
                testServerEventsInLonerClient_port = startBridgeServer(ports[0]);
            } catch (IOException e) {
                getLogWriter().error(e);
                fail(e.getMessage());
            }
            return basicGetSystem().getDistributedMember();
        }
    };
    DistributedMember serverMember = (DistributedMember) vm0.invoke(createBridgeServer);
    String serverMemberId = serverMember.getId();
    // gather details for later creation of pool...
    assertEquals(ports[0], (int) vm0.invoke("getServerPort", () -> UniversalMembershipListenerAdapterDUnitTest.getTestServerEventsInLonerClient_port()));
    System.out.println("[testServerEventsInLonerClient] ports[0]=" + ports[0]);
    System.out.println("[testServerEventsInLonerClient] serverMemberId=" + serverMemberId);
    System.out.println("[testServerEventsInLonerClient] serverMember=" + serverMember);
    // create region which connects to bridge server
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(Scope.LOCAL);
    ClientServerTestCase.configureConnectionPool(factory, NetworkUtils.getServerHostName(host), ports, false, -1, -1, null);
    createRegion(name, factory.create());
    assertNotNull(getRootRegion().getSubregion(name));
    Awaitility.await("wait for join").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
        synchronized (adapter) {
            return firedAdapter[JOINED];
        }
    });
    Awaitility.await("wait for join").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
        synchronized (bridgeListener) {
            return firedBridge[JOINED];
        }
    });
    System.out.println("[testServerEventsInLonerClient] assert client detected server join");
    assertArrayFalse(firedAdapterDuplicate);
    assertArrayFalse(firedBridgeDuplicate);
    assertTrue(firedBridge[JOINED]);
    assertNotNull(memberBridge[JOINED]);
    assertNotNull(memberIdBridge[JOINED]);
    assertFalse(isClientBridge[JOINED]);
    assertFalse(firedBridge[LEFT]);
    assertNull(memberBridge[LEFT]);
    assertNull(memberIdBridge[LEFT]);
    assertFalse(isClientBridge[LEFT]);
    assertFalse(firedBridge[CRASHED]);
    assertNull(memberBridge[CRASHED]);
    assertNull(memberIdBridge[CRASHED]);
    assertFalse(isClientBridge[CRASHED]);
    resetArraysForTesting(firedBridge, memberBridge, memberIdBridge, isClientBridge);
    assertTrue(firedAdapter[JOINED]);
    assertNotNull(memberIdAdapter[JOINED]);
    assertFalse(isClientAdapter[JOINED]);
    assertFalse(firedAdapter[LEFT]);
    assertNull(memberAdapter[LEFT]);
    assertNull(memberIdAdapter[LEFT]);
    assertFalse(isClientAdapter[LEFT]);
    assertFalse(firedAdapter[CRASHED]);
    assertNull(memberAdapter[CRASHED]);
    assertNull(memberIdAdapter[CRASHED]);
    assertFalse(isClientAdapter[CRASHED]);
    resetArraysForTesting(firedAdapter, memberAdapter, memberIdAdapter, isClientAdapter);
    System.out.println("[testServerEventsInLonerClient] wait for client to fully connect");
    final String pl = getRootRegion().getSubregion(name).getAttributes().getPoolName();
    PoolImpl pi = (PoolImpl) PoolManager.find(pl);
    waitForClientToFullyConnect(pi);
    String expected = "java.io.IOException";
    String addExpected = "<ExpectedException action=add>" + expected + "</ExpectedException>";
    String removeExpected = "<ExpectedException action=remove>" + expected + "</ExpectedException>";
    String expected2 = "java.net.ConnectException";
    String addExpected2 = "<ExpectedException action=add>" + expected2 + "</ExpectedException>";
    String removeExpected2 = "<ExpectedException action=remove>" + expected2 + "</ExpectedException>";
    // LogWriter bgexecLogger =
    // new LocalLogWriter(LocalLogWriter.ALL_LEVEL, System.out);
    // bgexecLogger.info(addExpected);
    // bgexecLogger.info(addExpected2);
    LogWriter lw = getSystem().getLogWriter();
    lw.info(addExpected);
    lw.info(addExpected2);
    try {
        vm0.invoke(new SerializableRunnable("Disconnect bridge server") {

            public void run() {
                System.out.println("[testServerEventsInLonerClient] disconnect bridge server");
                closeCache();
            }
        });
        Awaitility.await("wait for server to leave").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
            synchronized (adapter) {
                return firedAdapter[LEFT] || firedAdapter[CRASHED];
            }
        });
        Awaitility.await("wait for server to leave").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
            synchronized (bridgeListener) {
                return firedBridge[LEFT] || firedBridge[CRASHED];
            }
        });
    } finally {
        // bgexecLogger.info(removeExpected);
        // bgexecLogger.info(removeExpected2);
        lw.info(removeExpected);
        lw.info(removeExpected2);
    }
    System.out.println("[testServerEventsInLonerClient] assert client detected server crashed");
    assertArrayFalse(firedAdapterDuplicate);
    assertArrayFalse(firedBridgeDuplicate);
    assertFalse(firedBridge[JOINED]);
    assertNull(memberIdBridge[JOINED]);
    assertNull(memberBridge[JOINED]);
    assertFalse(isClientBridge[JOINED]);
    assertFalse("Please update testServerEventsInLonerClient to handle memberLeft for BridgeServer.", firedBridge[LEFT]);
    assertNull(memberBridge[LEFT]);
    assertNull(memberIdBridge[LEFT]);
    assertFalse(isClientBridge[LEFT]);
    assertTrue(firedBridge[CRASHED]);
    assertNotNull(memberBridge[CRASHED]);
    assertNotNull(memberIdBridge[CRASHED]);
    assertFalse(isClientBridge[CRASHED]);
    resetArraysForTesting(firedBridge, memberBridge, memberIdBridge, isClientBridge);
    assertFalse(firedAdapter[JOINED]);
    assertNull(memberAdapter[JOINED]);
    assertNull(memberIdAdapter[JOINED]);
    assertFalse(isClientAdapter[JOINED]);
    assertFalse("Please update testServerEventsInLonerClient to handle BridgeServer LEFT", firedAdapter[LEFT]);
    assertNull(memberAdapter[LEFT]);
    assertNull(memberIdAdapter[LEFT]);
    assertFalse(isClientAdapter[LEFT]);
    // CRASHED fired by Bridge listener
    assertTrue(firedAdapter[CRASHED]);
    assertNotNull(memberAdapter[CRASHED]);
    assertNotNull(memberIdAdapter[CRASHED]);
    assertFalse(isClientAdapter[CRASHED]);
    resetArraysForTesting(firedAdapter, memberAdapter, memberIdAdapter, isClientAdapter);
    // reconnect bridge client to test for crashed event
    serverMember = (DistributedMember) vm0.invoke(createBridgeServer);
    serverMemberId = serverMember.getId();
    // gather details for later creation of pool...
    assertEquals(ports[0], (int) vm0.invoke(() -> UniversalMembershipListenerAdapterDUnitTest.getTestServerEventsInLonerClient_port()));
    serverMember = vm0.invoke(() -> UniversalMembershipListenerAdapterDUnitTest.getDistributedMember());
    System.out.println("[testServerEventsInLonerClient] ports[0]=" + ports[0]);
    System.out.println("[testServerEventsInLonerClient] serverMemberId=" + serverMemberId);
    System.out.println("[testServerEventsInLonerClient] serverMember=" + serverMember);
    Awaitility.await("wait for join").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
        synchronized (adapter) {
            return firedAdapter[JOINED];
        }
    });
    Awaitility.await("wait for join").atMost(ASYNC_EVENT_WAIT_MILLIS, TimeUnit.MILLISECONDS).until(() -> {
        synchronized (bridgeListener) {
            return firedBridge[JOINED];
        }
    });
    System.out.println("[testServerEventsInLonerClient] assert client detected server re-join");
    assertArrayFalse(firedAdapterDuplicate);
    assertArrayFalse(firedBridgeDuplicate);
    assertTrue(firedBridge[JOINED]);
    assertNotNull(memberBridge[JOINED]);
    assertNotNull(memberIdBridge[JOINED]);
    assertFalse(isClientBridge[JOINED]);
    assertFalse(firedBridge[LEFT]);
    assertNull(memberBridge[LEFT]);
    assertNull(memberIdBridge[LEFT]);
    assertFalse(isClientBridge[LEFT]);
    assertFalse(firedBridge[CRASHED]);
    assertNull(memberBridge[CRASHED]);
    assertNull(memberIdBridge[CRASHED]);
    assertFalse(isClientBridge[CRASHED]);
    resetArraysForTesting(firedBridge, memberBridge, memberIdBridge, isClientBridge);
    assertTrue(firedAdapter[JOINED]);
    assertNotNull(memberAdapter[JOINED]);
    assertNotNull(memberIdAdapter[JOINED]);
    assertFalse(isClientAdapter[JOINED]);
    assertFalse(firedAdapter[LEFT]);
    assertNull(memberAdapter[LEFT]);
    assertNull(memberIdAdapter[LEFT]);
    assertFalse(isClientAdapter[LEFT]);
    assertFalse(firedAdapter[CRASHED]);
    assertNull(memberAdapter[CRASHED]);
    assertNull(memberIdAdapter[CRASHED]);
    assertFalse(isClientAdapter[CRASHED]);
    resetArraysForTesting(firedAdapter, memberAdapter, memberIdAdapter, isClientAdapter);
}