Search in sources :

Example 6 with Condition

use of com.nokia.dempsy.TestUtils.Condition in project Dempsy by Dempsy.

the class TestZookeeperClusterResilience method testSessionExpired.

@Test
public void testSessionExpired() throws Throwable {
    // now lets startup the server.
    ZookeeperTestServer server = null;
    ZookeeperSession session = null;
    try {
        server = new ZookeeperTestServer();
        server.start();
        // the createExpireSessionClient actually results in a Disconnected/SyncConnected rotating events.
        // ... so we need to filter those out since it will result in a callback.
        session = new ZookeeperSession("127.0.0.1:" + port, 5000);
        final ClusterId clusterId = new ClusterId(appname, "testSessionExpired");
        createClusterLevel(clusterId, session);
        TestWatcher callback = new TestWatcher(session) {

            @Override
            public void process() {
                try {
                    called.set(true);
                    logger.trace("process called on TestWatcher.");
                    session.exists(clusterId.asPath(), this);
                    session.getSubdirs(clusterId.asPath(), this);
                } catch (ClusterInfoException cie) {
                    throw new RuntimeException(cie);
                }
            }
        };
        // now see if the cluster works.
        // this registers the session with the callback as the Watcher
        callback.process();
        // now reset the condition
        callback.called.set(false);
        ZookeeperTestServer.forceSessionExpiration(session.zkref.get());
        // we should see the session expiration in a callback
        assertTrue(poll(5000, callback, new Condition<TestWatcher>() {

            @Override
            public boolean conditionMet(TestWatcher o) {
                return o.called.get();
            }
        }));
        // and eventually a reconnect
        assertTrue(poll(5000, callback, new Condition<TestWatcher>() {

            @Override
            public boolean conditionMet(TestWatcher o) {
                try {
                    o.process();
                    return true;
                } catch (Throwable th) {
                    return false;
                }
            }
        }));
        createClusterLevel(clusterId, session);
        assertTrue(session.exists(clusterId.asPath(), callback));
    } finally {
        if (server != null)
            server.shutdown();
        if (session != null)
            session.stop();
    }
}
Also used : Condition(com.nokia.dempsy.TestUtils.Condition) ClusterId(com.nokia.dempsy.config.ClusterId) ClusterInfoException(com.nokia.dempsy.cluster.ClusterInfoException) Test(org.junit.Test)

Example 7 with Condition

use of com.nokia.dempsy.TestUtils.Condition in project Dempsy by Dempsy.

the class TestZookeeperClusterResilience method testRecoverWithIOException.

@Test
public void testRecoverWithIOException() throws Throwable {
    // now lets startup the server.
    ZookeeperTestServer server = null;
    ZookeeperSession sessiong = null;
    try {
        server = new ZookeeperTestServer();
        server.start();
        final ZookeeperSession session = new ZookeeperSession("127.0.0.1:" + port, 5000) {

            @Override
            protected ZooKeeper makeZooKeeperClient(String connectString, int sessionTimeout) throws IOException {
                if (forceIOException.get()) {
                    forceIOExceptionLatch.countDown();
                    throw new IOException("Fake IO Problem.");
                }
                return super.makeZooKeeperClient(connectString, sessionTimeout);
            }
        };
        sessiong = session;
        final ClusterId clusterId = new ClusterId(appname, "testRecoverWithIOException");
        TestUtils.createClusterLevel(clusterId, session);
        TestWatcher callback = new TestWatcher(session) {

            @Override
            public void process() {
                try {
                    session.getSubdirs(clusterId.asPath(), this);
                    called.set(true);
                } catch (ClusterInfoException cie) {
                    throw new RuntimeException(cie);
                }
            }
        };
        callback.process();
        // force the ioexception to happen
        forceIOException.set(true);
        ZookeeperTestServer.forceSessionExpiration(session.zkref.get());
        // now in the background it should be retrying but hosed.
        assertTrue(forceIOExceptionLatch.await(baseTimeoutMillis * 3, TimeUnit.MILLISECONDS));
        // now the getActiveSlots call should fail since i'm preventing the recovery by throwing IOExceptions
        assertTrue(TestUtils.poll(baseTimeoutMillis, clusterId, new Condition<ClusterId>() {

            @Override
            public boolean conditionMet(ClusterId o) throws Throwable {
                try {
                    session.mkdir(o.asPath() + "/join-1", null, DirMode.EPHEMERAL);
                    return false;
                } catch (ClusterInfoException e) {
                    return true;
                }
            }
        }));
        // reset the callbacker ...
        callback.called.set(false);
        // now we should allow the code to proceed.
        forceIOException.set(false);
        // wait for the callback
        assertTrue(poll(baseTimeoutMillis, callback, new Condition<TestWatcher>() {

            @Override
            public boolean conditionMet(TestWatcher o) {
                return o.called.get();
            }
        }));
        // this should eventually recover.
        assertTrue(TestUtils.poll(baseTimeoutMillis, clusterId, new Condition<ClusterId>() {

            @Override
            public boolean conditionMet(ClusterId o) throws Throwable {
                try {
                    TestUtils.createClusterLevel(o, session);
                    session.mkdir(o.asPath() + "/join-1", null, DirMode.EPHEMERAL);
                    return true;
                } catch (ClusterInfoException e) {
                    return false;
                }
            }
        }));
        session.getSubdirs(clusterId.asPath(), callback);
        // And join should work
        // And join should work
        assertTrue(TestUtils.poll(baseTimeoutMillis, clusterId, new Condition<ClusterId>() {

            @Override
            public boolean conditionMet(ClusterId o) throws Throwable {
                try {
                    session.mkdir(o.asPath() + "/join-1", null, DirMode.EPHEMERAL);
                    return true;
                } catch (ClusterInfoException e) {
                }
                return false;
            }
        }));
    } finally {
        if (server != null)
            server.shutdown();
        if (sessiong != null)
            sessiong.stop();
    }
}
Also used : Condition(com.nokia.dempsy.TestUtils.Condition) ClusterId(com.nokia.dempsy.config.ClusterId) ClusterInfoException(com.nokia.dempsy.cluster.ClusterInfoException) IOException(java.io.IOException) Test(org.junit.Test)

Example 8 with Condition

use of com.nokia.dempsy.TestUtils.Condition in project Dempsy by Dempsy.

the class ZookeeperTestServer method forceSessionExpiration.

public static void forceSessionExpiration(ZooKeeper origZk) throws Throwable {
    TestUtils.Condition<ZooKeeper> condition = new TestUtils.Condition<ZooKeeper>() {

        @Override
        public boolean conditionMet(ZooKeeper o) throws Throwable {
            try {
                return (o.getState() == ZooKeeper.States.CONNECTED) && o.exists("/", true) != null;
            } catch (KeeperException ke) {
                return false;
            }
        }
    };
    assertTrue(TestUtils.poll(5000, origZk, condition));
    boolean done = false;
    while (!done) {
        long sessionid = origZk.getSessionId();
        byte[] pw = origZk.getSessionPasswd();
        KWatcher kwatcher = new KWatcher();
        ZooKeeper killer = new ZooKeeper("127.0.0.1:" + port, 5000, kwatcher, sessionid, pw);
        kwatcher.connection.set(killer);
        // wait until we get a close
        boolean calledBack = TestUtils.poll(5000, kwatcher, new Condition<KWatcher>() {

            @Override
            public boolean conditionMet(KWatcher o) throws Throwable {
                return o.closed.get();
            }
        });
        if (!calledBack)
            killer.close();
        final AtomicBoolean isExpired = new AtomicBoolean(false);
        ZooKeeper check = new ZooKeeper("127.0.0.1:" + port, 5000, new Watcher() {

            @Override
            public void process(WatchedEvent event) {
                if (event.getState() == Watcher.Event.KeeperState.Expired)
                    isExpired.set(true);
            }
        }, sessionid, pw);
        done = TestUtils.poll(5000, isExpired, new Condition<AtomicBoolean>() {

            public boolean conditionMet(AtomicBoolean o) {
                return o.get();
            }
        });
        check.close();
    }
}
Also used : Condition(com.nokia.dempsy.TestUtils.Condition) Watcher(org.apache.zookeeper.Watcher) WatchedEvent(org.apache.zookeeper.WatchedEvent) TestUtils(com.nokia.dempsy.TestUtils) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ZooKeeper(org.apache.zookeeper.ZooKeeper) KeeperException(org.apache.zookeeper.KeeperException)

Example 9 with Condition

use of com.nokia.dempsy.TestUtils.Condition in project Dempsy by Dempsy.

the class TestDempsy method testFailedMessageHandlingWithKeyStore.

@Test
public void testFailedMessageHandlingWithKeyStore() throws Throwable {
    final AtomicBoolean currentActivateCheckedException = new AtomicBoolean(false);
    Checker checker = new Checker() {

        @Override
        public void check(ApplicationContext context) throws Throwable {
            // start things and verify that the init method was called
            Dempsy dempsy = (Dempsy) context.getBean("dempsy");
            TestMp mp = (TestMp) getMp(dempsy, "test-app", "test-cluster1");
            // verify we haven't called it again, not that there's really
            // a way to given the code
            assertEquals(1, mp.startCalls.get());
            // make sure that there are no Mps
            MetricGetters statsCollector = (MetricGetters) dempsy.getCluster(new ClusterId("test-app", "test-cluster1")).getNodes().get(0).getStatsCollector();
            Thread.sleep(10);
            assertEquals(0, statsCollector.getMessageProcessorsCreated());
            mp.failActivation.set("test1");
            TestAdaptor adaptor = (TestAdaptor) context.getBean("adaptor");
            // this causes the container to clone the Mp
            adaptor.pushMessage(new TestMessage("test1"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.cloneCalls.get() == 1;
                }
            }));
            Thread.sleep(100);
            assertEquals(0, statsCollector.getMessageProcessorsCreated());
            mp.failActivation.set(null);
            KeySourceImpl.pause.countDown();
            // instead of the latch we are going to poll for the correct result
            // wait for it to be received.
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.cloneCalls.get() == 3;
                }
            }));
            assertTrue(poll(baseTimeoutMillis, statsCollector, new Condition<MetricGetters>() {

                @Override
                public boolean conditionMet(MetricGetters mg) {
                    return mg.getMessageProcessorsCreated() == 2;
                }
            }));
            adaptor.pushMessage(new TestMessage("test1"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 1;
                }
            }));
            adaptor.pushMessage(new TestMessage("test2"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 2;
                }
            }));
            adaptor.pushMessage(new TestMessage("test1"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 3;
                }
            }));
            adaptor.pushMessage(new TestMessage("test2"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 4;
                }
            }));
            adaptor.pushMessage(new TestMessage("test1"));
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 5;
                }
            }));
            adaptor.pushMessage(new TestMessage("test2"));
            // instead of the latch we are going to poll for the correct result
            // wait for it to be received.
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.handleCalls.get() == 6;
                }
            }));
            Thread.sleep(100);
            assertEquals(6, mp.handleCalls.get());
            assertEquals(3, mp.cloneCalls.get());
            assertEquals(2, statsCollector.getMessageProcessorsCreated());
            // prepare for the next run
            KeySourceImpl.pause = new CountDownLatch(1);
        }

        public String toString() {
            return "testFailedMessageHandlingWithKeyStore";
        }

        public void setup() {
            KeySourceImpl.pause = new CountDownLatch(1);
            TestMp.activateCheckedException = currentActivateCheckedException.get();
        }
    };
    // make sure both exceptions are handled since the logic in the container
    // actually varies depending on whether or not the exception is checked or not.
    currentActivateCheckedException.set(true);
    runAllCombinations("SinglestageWithKeyStoreApplicationActx.xml", checker);
    currentActivateCheckedException.set(false);
    runAllCombinations("SinglestageWithKeyStoreAndExecutorApplicationActx.xml", checker);
}
Also used : Condition(com.nokia.dempsy.TestUtils.Condition) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClassPathXmlApplicationContext(org.springframework.context.support.ClassPathXmlApplicationContext) ApplicationContext(org.springframework.context.ApplicationContext) ClusterId(com.nokia.dempsy.config.ClusterId) CountDownLatch(java.util.concurrent.CountDownLatch) MetricGetters(com.nokia.dempsy.monitoring.coda.MetricGetters) Test(org.junit.Test)

Example 10 with Condition

use of com.nokia.dempsy.TestUtils.Condition in project Dempsy by Dempsy.

the class TestDempsy method testOverlappingKeyStoreCalls.

@Test
public void testOverlappingKeyStoreCalls() throws Throwable {
    Checker checker = new Checker() {

        @Override
        public void check(ApplicationContext context) throws Throwable {
            // wait until the KeySourceImpl has been created
            assertTrue(poll(baseTimeoutMillis, null, new Condition<Object>() {

                @Override
                public boolean conditionMet(Object mp) {
                    return KeySourceImpl.lastCreated != null;
                }
            }));
            final KeySourceImpl.KSIterable firstCreated = KeySourceImpl.lastCreated;
            // start things and verify that the init method was called
            Dempsy dempsy = (Dempsy) context.getBean("dempsy");
            TestMp mp = (TestMp) getMp(dempsy, "test-app", "test-cluster1");
            Dempsy.Application.Cluster c = dempsy.getCluster(new ClusterId("test-app", "test-cluster1"));
            assertNotNull(c);
            Dempsy.Application.Cluster.Node node = c.getNodes().get(0);
            assertNotNull(node);
            MpContainer container = node.getMpContainer();
            // let it go and wait until there's a few keys.
            firstCreated.m_pause.countDown();
            // as the KeySource iterates, this will increase
            assertTrue(poll(baseTimeoutMillis, mp, new Condition<TestMp>() {

                @Override
                public boolean conditionMet(TestMp mp) {
                    return mp.cloneCalls.get() > 10000;
                }
            }));
            // prepare the next countdown latch
            // just let the 2nd one go
            KeySourceImpl.pause = new CountDownLatch(0);
            // I want the next one to stop at 2
            KeySourceImpl.infinite = false;
            // Now force another call while the first is running
            container.keyspaceResponsibilityChanged(node.strategyInbound, false, true);
            // wait until the second one is created
            assertTrue(poll(baseTimeoutMillis, null, new Condition<Object>() {

                @Override
                public boolean conditionMet(Object mp) {
                    return KeySourceImpl.lastCreated != null && firstCreated != KeySourceImpl.lastCreated;
                }
            }));
            // now the first one should be done and therefore no longer incrementing.
            String lastKeyOfFirstCreated = firstCreated.lastKey;
            // and the second one should be done also and stopped at 2.
            final KeySourceImpl.KSIterable secondCreated = KeySourceImpl.lastCreated;
            assertTrue(firstCreated != secondCreated);
            assertTrue(poll(baseTimeoutMillis, null, new Condition<Object>() {

                @Override
                public boolean conditionMet(Object mp) {
                    return "test2".equals(secondCreated.lastKey);
                }
            }));
            Thread.sleep(50);
            // make sure the first one isn't still moving on
            assertEquals(lastKeyOfFirstCreated, firstCreated.lastKey);
            assertEquals("test2", secondCreated.lastKey);
            // prepare for the next run
            KeySourceImpl.pause = new CountDownLatch(1);
            KeySourceImpl.infinite = true;
            KeySourceImpl.lastCreated = null;
        }

        public String toString() {
            return "testOverlappingKeyStoreCalls";
        }

        public void setup() {
            KeySourceImpl.pause = new CountDownLatch(1);
            KeySourceImpl.infinite = true;
            KeySourceImpl.lastCreated = null;
        }
    };
    runAllCombinations("SinglestageWithKeyStoreApplicationActx.xml", checker);
    runAllCombinations("SinglestageWithKeyStoreAndExecutorApplicationActx.xml", checker);
}
Also used : Condition(com.nokia.dempsy.TestUtils.Condition) MpContainer(com.nokia.dempsy.container.MpContainer) ClusterId(com.nokia.dempsy.config.ClusterId) CountDownLatch(java.util.concurrent.CountDownLatch) ClassPathXmlApplicationContext(org.springframework.context.support.ClassPathXmlApplicationContext) ApplicationContext(org.springframework.context.ApplicationContext) Node(com.nokia.dempsy.Dempsy.Application.Cluster.Node) Test(org.junit.Test)

Aggregations

Condition (com.nokia.dempsy.TestUtils.Condition)15 ClusterId (com.nokia.dempsy.config.ClusterId)13 Test (org.junit.Test)13 ClassPathXmlApplicationContext (org.springframework.context.support.ClassPathXmlApplicationContext)10 ClusterInfoException (com.nokia.dempsy.cluster.ClusterInfoException)6 Dempsy (com.nokia.dempsy.Dempsy)5 ClusterInfoSession (com.nokia.dempsy.cluster.ClusterInfoSession)5 ZooKeeper (org.apache.zookeeper.ZooKeeper)5 ApplicationContext (org.springframework.context.ApplicationContext)5 MetricGetters (com.nokia.dempsy.monitoring.coda.MetricGetters)4 ClusterInfoSessionFactory (com.nokia.dempsy.cluster.ClusterInfoSessionFactory)3 MyMp (com.nokia.dempsy.cluster.zookeeper.FullApplication.MyMp)3 ApplicationDefinition (com.nokia.dempsy.config.ApplicationDefinition)3 StatsCollector (com.nokia.dempsy.monitoring.StatsCollector)3 Node (com.nokia.dempsy.Dempsy.Application.Cluster.Node)2 MyRankMp (com.nokia.dempsy.cluster.zookeeper.FullApplication.MyRankMp)2 ClusterDefinition (com.nokia.dempsy.config.ClusterDefinition)2 HashMap (java.util.HashMap)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2