Search in sources :

Example 81 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestDatabusRelayMain method testClient.

private void testClient(int relayPort, int fetchSize, long scn, HttpResponseHandler handler) throws Exception {
    Checkpoint ckpt = Checkpoint.createOnlineConsumptionCheckpoint(scn);
    //TODO why is this needed
    //ckpt.setCatchupSource("foo");
    String uristr = "/stream?sources=105&output=json&size=" + fetchSize + "&streamFromLatestScn=false&checkPoint=" + ckpt.toString();
    ClientBootstrap bootstrap = new ClientBootstrap(new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), Executors.newCachedThreadPool()));
    bootstrap.setPipelineFactory(new HttpClientPipelineFactory(handler));
    ChannelFuture future = bootstrap.connect(new InetSocketAddress("localhost", relayPort));
    Channel channel = future.awaitUninterruptibly().getChannel();
    Assert.assertTrue(future.isSuccess(), "Cannot connect to relay at localhost:" + relayPort);
    HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uristr);
    request.setHeader(HttpHeaders.Names.HOST, "localhost");
    channel.write(request);
    channel.getCloseFuture().awaitUninterruptibly();
}
Also used : ChannelFuture(org.jboss.netty.channel.ChannelFuture) HttpRequest(org.jboss.netty.handler.codec.http.HttpRequest) DefaultHttpRequest(org.jboss.netty.handler.codec.http.DefaultHttpRequest) Checkpoint(com.linkedin.databus.core.Checkpoint) NioClientSocketChannelFactory(org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory) ClientBootstrap(org.jboss.netty.bootstrap.ClientBootstrap) InetSocketAddress(java.net.InetSocketAddress) DefaultHttpRequest(org.jboss.netty.handler.codec.http.DefaultHttpRequest) Channel(org.jboss.netty.channel.Channel)

Example 82 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockRemoteExceptionHandler method testRelayTransition.

@Test(groups = { "small", "functional" })
public void testRelayTransition() throws Exception {
    final Logger log = Logger.getLogger("TestRelayPullThread.testRelayTransition");
    log.info("---------- start ---------------");
    // Test Case : Initial State Check
    {
        RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
        //RelayPullThread relayPuller = createRelayPullThread(false, false);
        RelayPullThread relayPuller = bldr.createRelayPullThread();
        Assert.assertEquals(relayPuller.getConnectionState().getStateId(), StateId.INITIAL, "Initial State Check");
        //Let the show begin
        Thread relayPullerThread = new Thread(relayPuller);
        relayPullerThread.setDaemon(false);
        relayPullerThread.start();
        relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
        try {
            Thread.sleep(500);
        } catch (InterruptedException ie) {
        }
        for (int i = 0; i < 100; i++) {
            try {
                Thread.sleep(5);
                Assert.assertTrue(relayPuller.getConnectionState().getStateId() != StateId.INITIAL, "StateId can never be INITIAL once started");
            } catch (InterruptedException ie) {
            }
        }
        //EasyMock.verify(mockConnFactory);
        relayPuller.enqueueMessage(LifecycleMessage.createShutdownMessage());
        relayPuller.awaitShutdown();
    }
    // Main Transition Test (Without Server-Set Change)
    {
        // PICK_SERVER - Happy Path
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
        }
        // PICK_SERVER  - Relays exhausted
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(true, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.PICK_SERVER, "SUSPEND_ON_ERROR");
        }
        // PICK_SERVER  - No Servers
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            relayPuller.getServers().clear();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.PICK_SERVER, "SUSPEND_ON_ERROR");
        }
        // Request_Sources to Re Sources_Request_Sent
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
        }
        // Request_Sources to Sources_Response_Success
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
        }
        // Sources_Response_Success - Happy Path
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            validateConnState(connState);
            String expSubsListStr = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"ANY\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":1,\"name\":\"source1\"},\"id\":-1}}]";
            String expSourcesIdListStr = "1";
            Assert.assertEquals(connState.getSourcesIdListString(), expSourcesIdListStr, "SourcesId Added");
            String subsListStr = connState.getSubsListString();
            Assert.assertEquals(subsListStr, expSubsListStr);
        }
        // Sources_Response_Success - When source not found in server
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            connState.getSourcesNameMap().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.PICK_SERVER);
            validateConnState(connState);
        }
        // Request_Register to Register_Request_Sent
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            validateConnState(connState);
        }
        // Request_Register to Register_Response_success
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
        }
        // Register_Response_Success : Error Case
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            connState.getSourcesSchemas().clear();
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.PICK_SERVER);
        }
        // Register_Response_Success to Request Stream
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            Assert.assertEquals(relayPuller.getConnectionState().isSCNRegress(), false, "SCN Regress check");
        }
        // Register_Response_Success to Request Stream, when partially consumed window
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            Checkpoint cp = new Checkpoint();
            cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
            cp.setWindowScn(100L);
            cp.setWindowOffset(20);
            cp.setPrevScn(80L);
            relayPuller.getConnectionState().setCheckpoint(cp);
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            Assert.assertEquals(relayPuller.getConnectionState().getCheckpoint().getWindowScn(), 80L, "WindowSCN check");
            Assert.assertEquals(relayPuller.getConnectionState().getCheckpoint().getWindowOffset(), new Long(-1), "WindowOffset check");
            Assert.assertEquals(relayPuller.getConnectionState().isSCNRegress(), true, "SCN Regress check");
        }
        // Register_Response_Success, No PrevSCN for partially consumed window
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            Checkpoint cp = new Checkpoint();
            cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
            cp.setWindowScn(100L);
            cp.setWindowOffset(20);
            relayPuller.getConnectionState().setCheckpoint(cp);
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REGISTER_RESPONSE_SUCCESS, "SUSPEND_ON_ERROR");
        }
        // Register_Response_Success to Bootstrap ( when checkpoint is Bootstrap_SnapShot
        {
            //RelayPullThread relayPuller = createRelayPullThread(false, false, true);
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Is Relay FellOff");
            Checkpoint cp = new Checkpoint();
            cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
            connState.setCheckpoint(cp);
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.BOOTSTRAP);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            Assert.assertEquals(relayPuller.getConnectionState().isSCNRegress(), false, "SCN Regress check");
        }
        // Register_Response_Success to Bootstrap ( when checkpoint is Bootstrap_Catchup
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Is Relay FellOff");
            Checkpoint cp = new Checkpoint();
            cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
            connState.setCheckpoint(cp);
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.BOOTSTRAP);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            Assert.assertEquals(relayPuller.getConnectionState().isSCNRegress(), false, "SCN Regress check");
        }
        // Request_Stream to Request_Stream_Sent
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            relayPuller.getMessageQueue().clear();
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            connState.switchToRegisterSuccess(conn.getRegisterResponse(), conn.getRegisterResponse(), conn.getRegisterMetadataResponse());
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SENT, "");
            validateConnState(connState);
        }
        // Request_Stream to Stream_Response_Success
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
        }
        //Stream_Request_Success to Stream_Request_Done
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE);
        }
        //Stream_Request_Success : ScnNotFoundException but retries set to 5 and  bootstrap enabled
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName()).setNumRetriesOnFellOff(5);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            for (int i = 1; i <= 6; i++) {
                System.out.println("Iteration :" + i);
                testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
                relayPuller.getMessageQueue().clear();
                if (i < 6) {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 6 - i, "Retry State");
                } else {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
                    //reset
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 5, "Retry State");
                }
                relayPuller.getMessageQueue().clear();
            }
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "RelayFellOff State");
        }
        //Stream_Request_Success : ScnNotFoundException and  bootstrap enabled
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
        }
        //Stream_Request_Success : ScnNotFoundException but retries set to 5, bootstrap disabled and readLatestScnOnFallOff enabled.
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(false).setReadLatestScnEnabled(true).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName()).setNumRetriesOnFellOff(5);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            for (int i = 1; i <= 6; i++) {
                testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
                relayPuller.getMessageQueue().clear();
                if (i < 6) {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 6 - i, "Retry State");
                } else {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE);
                    //reset
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 5, "Retry State");
                }
                relayPuller.getMessageQueue().clear();
            }
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            Assert.assertEquals(conn.isReadFromLatestScn(), true, "ReadFromLatestScn set");
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "RelayFellOff State");
        }
        //Stream_Request_Success : ScnNotFoundException, bootstrap disabled and readLatestScnOnFallOff enabled.
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(false).setReadLatestScnEnabled(true).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE);
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            Assert.assertEquals(conn.isReadFromLatestScn(), true, "ReadFromLatestScn set");
        }
        //Stream_Request_Success : ScnNotFoundException but retries set to 5, bootstrap disabled and readLatestScnOnFallOff disabled.
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(false).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName()).setNumRetriesOnFellOff(5);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            for (int i = 1; i <= 6; i++) {
                testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
                relayPuller.getMessageQueue().clear();
                testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
                relayPuller.getMessageQueue().clear();
                if (i < 6) {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 6 - i, "Retry State");
                } else {
                    testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR");
                    //reset
                    Assert.assertEquals(relayPuller.getRetryonFallOff().getRemainingRetriesNum(), 5, "Retry State");
                }
                relayPuller.getMessageQueue().clear();
            }
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "RelayFellOff State");
        }
        //Stream_Request_Success : ScnNotFoundException, bootstrap disabled and readLatestScnOnFallOff disabled.
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(false).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR");
        }
        //Stream_Request_Success : Non-ScnNotFoundException
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName("DummyError");
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_ERROR);
        }
        //Stream_Request_Done to Request_Stream
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_RESPONSE_DONE, StateId.REQUEST_STREAM);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Is Relay FellOff");
        }
        // Bootstrap to Bootstrap_Requested
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
        }
        //Exception while doBootstrap()
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(true).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP, "SUSPEND_ON_ERROR");
        }
        // Bootstrap Failed : Case 1
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
            BootstrapResultMessage msg = BootstrapResultMessage.createBootstrapFailedMessage(new Exception("Dummy"));
            doExecuteAndChangeState(relayPuller, msg);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP, "BOOTSTRAP_REQUESTED to BOOTSTRAP");
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [BOOTSTRAP]", "Queue :BOOTSTRAP_REQUESTED to BOOTSTRAP");
        }
        // Bootstrap Failed : Case 2
        {
            //RelayPullThread relayPuller = createRelayPullThread(false, false, true, false, false, true, ScnNotFoundException.class.getName());
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
            BootstrapResultMessage msg = BootstrapResultMessage.createBootstrapCompleteMessage(null);
            doExecuteAndChangeState(relayPuller, msg);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP, "BOOTSTRAP_REQUESTED to BOOTSTRAP");
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [BOOTSTRAP]", "Queue :BOOTSTRAP_REQUESTED to BOOTSTRAP");
        }
        // Bootstrap Success
        {
            //RelayPullThread relayPuller = createRelayPullThread(false, false, true, false, false, true, ScnNotFoundException.class.getName());
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Is Relay FellOff");
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
            BootstrapResultMessage msg = BootstrapResultMessage.createBootstrapCompleteMessage(new Checkpoint());
            doExecuteAndChangeState(relayPuller, msg);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Is Relay FellOff");
            Assert.assertEquals(connState.getStateId(), StateId.REQUEST_STREAM, "BOOTSTRAP_REQUESTED to REQUEST_STREAM");
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [REQUEST_STREAM]", "Queue :BOOTSTRAP_REQUESTED to REQUEST_STREAM");
        }
        // Error States to Pick_Server
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            connState.switchToSourcesRequestError();
            testTransitionCase(relayPuller, StateId.SOURCES_REQUEST_ERROR, StateId.PICK_SERVER);
            relayPuller.getMessageQueue().clear();
            connState.switchToSourcesResponseError();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_ERROR, StateId.PICK_SERVER);
            relayPuller.getMessageQueue().clear();
            connState.switchToRegisterRequestError();
            testTransitionCase(relayPuller, StateId.REGISTER_REQUEST_ERROR, StateId.PICK_SERVER);
            relayPuller.getMessageQueue().clear();
            connState.switchToRegisterResponseError();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_ERROR, StateId.PICK_SERVER);
            relayPuller.getMessageQueue().clear();
            connState.switchToStreamRequestError();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_ERROR, StateId.PICK_SERVER);
            relayPuller.getMessageQueue().clear();
            connState.switchToStreamResponseError();
            testTransitionCase(relayPuller, StateId.STREAM_RESPONSE_ERROR, StateId.PICK_SERVER);
        }
    }
    log.info("--------- end -------------");
}
Also used : ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) Checkpoint(com.linkedin.databus.core.Checkpoint) InvalidEventException(com.linkedin.databus.core.InvalidEventException) ScnNotFoundException(com.linkedin.databus.core.ScnNotFoundException) DatabusException(com.linkedin.databus2.core.DatabusException) IOException(java.io.IOException) Checkpoint(com.linkedin.databus.core.Checkpoint) ScnNotFoundException(com.linkedin.databus.core.ScnNotFoundException) IdNamePair(com.linkedin.databus.core.util.IdNamePair) Test(org.testng.annotations.Test)

Example 83 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockRemoteExceptionHandler method testServerSetChanges.

@Test
public void testServerSetChanges() throws Exception {
    // Server-Set Change TestCases
    {
        Set<ServerInfo> expServerInfo = new TreeSet<ServerInfo>();
        expServerInfo.add(new ServerInfo("relay1", "ONLINE", new InetSocketAddress("localhost", 10001), "source1"));
        expServerInfo.add(new ServerInfo("relay2", "ONLINE", new InetSocketAddress("localhost", 10002), "source1"));
        expServerInfo.add(new ServerInfo("relay3", "ONLINE", new InetSocketAddress("localhost", 10003), "source1"));
        Set<ServerInfo> expServerInfo2 = new TreeSet<ServerInfo>(expServerInfo);
        expServerInfo2.add(new ServerInfo("relay4", "ONLINE", new InetSocketAddress("localhost", 10000), "source1"));
        Set<ServerInfo> expServerInfo3 = new TreeSet<ServerInfo>();
        expServerInfo3.add(new ServerInfo("relay4", "ONLINE", new InetSocketAddress("localhost", 10000), "source1"));
        // when on PICK_SERVER
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            relayPuller.enqueueMessage(connState);
            // ServerSetChange
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), -1, "Current Server Index");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), -1, "Current Server Index");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while Pick_Server");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while Pick_Server");
            }
        }
        // When on Request_Sources
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REQUEST_SOURCES, "ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [REQUEST_SOURCES]", "Queue :ServerSetChange while REQUEST_SOURCES");
            }
            // ServerSetChange when New Set excludes CurrentServer
            {
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on SOURCES-REQUEST-SENT and Response Successful
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while SOURCES_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while SOURCES_REQUEST_SENT");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff Check");
                // Now Response arrives
                List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
                sourcesResponse.add(new IdNamePair(1L, "source1"));
                connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
                testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
                validateConnState(connState);
            }
        }
        // When on SOURCES-REQUEST-SENT and SOURCES_RESPONSE_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while SOURCES_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while SOURCES_REQUEST_SENT");
            }
            // ServerSetChange when New Set excludes CurrentServer and SOURCES_RESPONSE_ERROR
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
                sourcesResponse.add(new IdNamePair(1L, "source1"));
                connState.switchToSourcesResponseError();
                testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_ERROR, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on SOURCES-REQUEST-SENT and SOURCES_REQUEST_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while SOURCES_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while SOURCES_REQUEST_SENT");
            }
            // ServerSetChange when New Set excludes CurrentServer and SOURCES_REQUEST_ERROR
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.SOURCES_REQUEST_SENT, "ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_SOURCES");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
                sourcesResponse.add(new IdNamePair(1L, "source1"));
                connState.switchToSourcesRequestError();
                testTransitionCase(relayPuller, StateId.SOURCES_REQUEST_ERROR, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on Request Register
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            validateConnState(connState);
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REQUEST_REGISTER, "ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [REQUEST_REGISTER]", "Queue :ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer
            {
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on REGISTER-REQUEST-SENT and REGISTER_RESPONSE_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REGISTER_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REGISTER_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                connState.switchToRegisterResponseError();
                testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_ERROR, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on REGISTER-REQUEST-SENT and REGISTER_REQUEST_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REGISTER_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REGISTER_REQUEST_SENT");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                connState.switchToSourcesRequestError();
                testTransitionCase(relayPuller, StateId.REGISTER_REQUEST_ERROR, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on REGISTER-REQUEST-SENT and Response Successful
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            validateConnState(connState);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REGISTER_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REGISTER_REQUEST_SENT");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REGISTER_REQUEST_SENT, "ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_REGISTER");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
                connState.switchToRegisterSuccess(conn.getRegisterResponse(), conn.getRegisterResponse(), conn.getRegisterMetadataResponse());
                testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // when on REQUEST_STREAM
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.REQUEST_STREAM, "ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [REQUEST_STREAM]", "Queue :ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer
            {
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on STREAM_REQUEST_SENT and response successful
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            relayPuller.getMessageQueue().clear();
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            connState.switchToRegisterSuccess(conn.getRegisterResponse(), conn.getRegisterResponse(), conn.getRegisterMetadataResponse());
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                conn = (MockRelayConnection) connState.getRelayConnection();
                connState.switchToStreamSuccess(conn.getStreamResponse());
                testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on STREAM_REQUEST_SENT and STREAM_RESPONSE_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            relayPuller.getMessageQueue().clear();
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            connState.switchToRegisterSuccess(conn.getRegisterResponse(), conn.getRegisterResponse(), conn.getRegisterMetadataResponse());
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and StreamResponse Error
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                connState.switchToStreamResponseError();
                testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When on STREAM_REQUEST_SENT and STREAM_REQUEST_ERROR
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, true);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_REQUEST_SENT, "");
            List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
            sourcesResponse.add(new IdNamePair(1L, "source1"));
            connState.switchToSourcesSuccess(sourcesResponse, _HOSTNAME, _SVCNAME);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            validateConnState(connState);
            relayPuller.getMessageQueue().clear();
            String subsListString = "[{\"physicalSource\":{\"uri\":\"databus:physical-source:ANY\",\"role\":\"MASTER\"},\"physicalPartition\":{\"id\":-1,\"name\":\"*\"},\"logicalPartition\":{\"source\":{\"id\":0,\"name\":\"source1\"},\"id\":-1}}]";
            String sourcesIdListString = "1";
            connState.switchToRequestSourcesSchemas(sourcesIdListString, subsListString);
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_REQUEST_SENT, "");
            relayPuller.getMessageQueue().clear();
            MockRelayConnection conn = (MockRelayConnection) connState.getRelayConnection();
            connState.switchToRegisterSuccess(conn.getRegisterResponse(), conn.getRegisterResponse(), conn.getRegisterMetadataResponse());
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SENT, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while STREAM_REQUEST_SENT");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and StreamRequest Error
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_REQUEST_SENT, "ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while REQUEST_STREAM");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                connState.switchToStreamRequestError();
                testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.PICK_SERVER);
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // when Stream_Response_done
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            RelayPullThread relayPuller = bldr.createFellOffRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE);
            Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            relayPuller.getConnectionState().setRelayFellOff(true);
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.STREAM_RESPONSE_DONE, "ServerSetChange while STREAM_RESPONSE_DONE");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [STREAM_RESPONSE_DONE]", "Queue :ServerSetChange while STREAM_RESPONSE_DONE");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer
            {
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while STREAM_RESPONSE_DONE");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while STREAM_RESPONSE_DONE");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When doBootstrap
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP, "ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [BOOTSTRAP]", "Queue :ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer
            {
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When doBootstrapRequested and bootstrap Successful
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP_REQUESTED, "ServerSetChange while BOOTSTRAP_REQUESTED");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while BOOTSTRAP_REQUESTED");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP_REQUESTED, "ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                BootstrapResultMessage msg = BootstrapResultMessage.createBootstrapCompleteMessage(new Checkpoint());
                doExecuteAndChangeState(relayPuller, msg);
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "BOOTSTRAP_REQUESTED to PICK_SERVER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :BOOTSTRAP_REQUESTED to REQUEST_STREAM");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // When doBootstrapRequested and bootstrap failed
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.BOOTSTRAP);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP_REQUESTED, "");
            // ServerSetChange when New Set includes CurrentServer
            {
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(true, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP_REQUESTED, "ServerSetChange while BOOTSTRAP_REQUESTED");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while BOOTSTRAP_REQUESTED");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
            }
            // ServerSetChange when New Set excludes CurrentServer and SuccessFul Response
            {
                int oldServerIndex = relayPuller.getCurrentServerIdx();
                ServerInfo oldServer = relayPuller.getCurentServer();
                Assert.assertEquals(relayPuller.getServers(), expServerInfo2, "Server Set");
                doExecuteAndChangeState(relayPuller, createSetServerMessage(false, relayPuller));
                Assert.assertEquals(relayPuller.getCurrentServerIdx(), oldServerIndex, "Current Server Index unchanged");
                Assert.assertEquals(relayPuller.getCurentServer(), oldServer, "Current Server unchanged");
                Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), true, "Tear Conn After Handling Response");
                Assert.assertEquals(connState.getStateId(), StateId.BOOTSTRAP_REQUESTED, "ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange while BOOTSTRAP");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), true, "Relay FellOff CHeck");
                // Now Response arrives
                BootstrapResultMessage msg = BootstrapResultMessage.createBootstrapFailedMessage(new RuntimeException("dummy"));
                doExecuteAndChangeState(relayPuller, msg);
                Assert.assertEquals(connState.getStateId(), StateId.PICK_SERVER, "BOOTSTRAP_REQUESTED to PICK_SERVER");
                Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :BOOTSTRAP_REQUESTED to REQUEST_STREAM");
                Assert.assertEquals(relayPuller.toTearConnAfterHandlingResponse(), false, "Tear Conn After Handling Response");
                Assert.assertEquals(relayPuller.getCurrentServerIdx() == -1, true, "Current Server Index undefined");
                Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server Null");
                Assert.assertEquals(relayPuller.getConnectionState().isRelayFellOff(), false, "Relay FellOff CHeck");
            }
        }
        // Server-Set-Change message when suspended_due_to_error
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName()).setNumRetriesOnFellOff(5);
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            ServerSetChangeMessage msg = createSetServerMessage(false, relayPuller);
            relayPuller.enqueueMessage(msg);
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [SOURCES_RESPONSE_SUCCESS, SET_SERVERS]", "Queue :ServerSetChange");
            doExecuteAndChangeState(relayPuller, LifecycleMessage.createSuspendOnErroMessage(new RuntimeException("null")));
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [SET_SERVERS]", "Queue :ServerSetChange");
            Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
            Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
            Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
            Assert.assertEquals(relayPuller.getComponentStatus().getStatus(), Status.SUSPENDED_ON_ERROR, "Suspended state check");
            connState.switchToRequestSourcesSchemas(null, null);
            relayPuller.getMessageQueue().clear();
            doExecuteAndChangeState(relayPuller, msg);
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [RESUME]", "Queue :ServerSetChange");
            Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server null");
            Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
            // on doResume verify if PICK_SERVER happens
            doExecuteAndChangeState(relayPuller, relayPuller.getMessageQueue().poll());
            Assert.assertEquals(relayPuller.getComponentStatus().getStatus(), Status.RUNNING, " state check");
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [PICK_SERVER]", "Queue :ServerSetChange");
        }
        // Server-Set-Change message when PAUSED
        {
            RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
            bldr.setBootstrapEnabled(true).setReadLatestScnEnabled(false).setReadDataThrowException(false).setReadDataException(true).setExceptionName(ScnNotFoundException.class.getName());
            RelayPullThread relayPuller = bldr.createRelayPullThread();
            relayPuller.getComponentStatus().start();
            ConnectionState connState = relayPuller.getConnectionState();
            connState.switchToPickServer();
            testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
            relayPuller.getMessageQueue().clear();
            testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
            ServerSetChangeMessage msg = createSetServerMessage(false, relayPuller);
            relayPuller.enqueueMessage(msg);
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [SOURCES_RESPONSE_SUCCESS, SET_SERVERS]", "Queue :ServerSetChange");
            doExecuteAndChangeState(relayPuller, LifecycleMessage.createPauseMessage());
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: [SET_SERVERS]", "Queue :ServerSetChange");
            Assert.assertEquals(relayPuller.getCurrentServerIdx() != -1, true, "Current Server Index defined");
            Assert.assertEquals(relayPuller.getCurentServer() != null, true, "Current Server not Null");
            Assert.assertEquals(relayPuller.getServers(), expServerInfo, "Server Set");
            Assert.assertEquals(relayPuller.getComponentStatus().getStatus(), Status.PAUSED, "PAUSED state check");
            connState.switchToRequestSourcesSchemas(null, null);
            relayPuller.getMessageQueue().clear();
            doExecuteAndChangeState(relayPuller, msg);
            Assert.assertEquals(relayPuller.getQueueListString(), "RelayPuller queue: []", "Queue :ServerSetChange");
            Assert.assertEquals(relayPuller.getCurentServer() == null, true, "Current Server null");
            Assert.assertEquals(relayPuller.getServers(), expServerInfo3, "Server Set");
        }
    }
}
Also used : Set(java.util.Set) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) ServerInfo(com.linkedin.databus.client.pub.ServerInfo) InetSocketAddress(java.net.InetSocketAddress) ArrayList(java.util.ArrayList) Checkpoint(com.linkedin.databus.core.Checkpoint) ScnNotFoundException(com.linkedin.databus.core.ScnNotFoundException) IdNamePair(com.linkedin.databus.core.util.IdNamePair) List(java.util.List) ArrayList(java.util.ArrayList) Test(org.testng.annotations.Test)

Example 84 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestGenericDispatcher method runPartialWindowCheckpointPersistence.

/**
     *
     * @param numEvents : number of events in buffer
     * @param maxWindowSize : window size expressed as number of events
     * @param numFailWindow : nth end-of-window that will fail
     * @throws Exception
     */
void runPartialWindowCheckpointPersistence(int numEvents, int maxWindowSize, int numFailWindow) throws Exception {
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    //setup consumer to fail on data callback at the nth event
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = numFailWindow;
    int numFailures = 1;
    //fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    //thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    //Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    //Assumption: generates events with  non-decreasing timestamps
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    //make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("OnlinePartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    //be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    //wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getWindowOffset().longValue(), -1L);
        //check if lastSeenCheckpoint by consumer is higher than scn persisted
        Assert.assertTrue(tConsumer.getLastSeenCheckpointScn() > cp.getWindowScn());
        //the latest event seen should be newer (or at least as new) as the checkpoint
        Assert.assertTrue(tConsumer.getLastTsInNanosOfEvent() >= tConsumer.getLastTsInNanosOfWindow());
        if (tConsumer.getLastSeenWindowScn() > 0) {
            Assert.assertEquals(cp.getWindowScn(), tConsumer.getLastSeenWindowScn());
            //check if the timestamp in checkpoint is the same as checkpoint of last completed window (ts of last event of the window)
            Assert.assertEquals(tConsumer.getLastTsInNanosOfWindow(), cp.getTsNsecs());
        } else {
            //not even one window was processed before error; expect uninitialized timestamp
            Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        }
    }
}
Also used : DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) DbusEventAppender(com.linkedin.databus.core.test.DbusEventAppender) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList) IdNamePair(com.linkedin.databus.core.util.IdNamePair) Vector(java.util.Vector) SelectingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer) DelegatingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.DelegatingDatabusCombinedConsumer) AbstractDatabusCombinedConsumer(com.linkedin.databus.client.consumer.AbstractDatabusCombinedConsumer) StreamConsumerCallbackFactory(com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory) DbusEvent(com.linkedin.databus.core.DbusEvent) MultiConsumerCallback(com.linkedin.databus.client.consumer.MultiConsumerCallback) DbusEventGenerator(com.linkedin.databus.core.test.DbusEventGenerator) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) Checkpoint(com.linkedin.databus.core.Checkpoint) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) Checkpoint(com.linkedin.databus.core.Checkpoint) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) Map(java.util.Map) HashMap(java.util.HashMap)

Example 85 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestGenericDispatcher method testShutdownBeforeRollback.

@Test(groups = { "small", "functional" })
public /**
     *
     * 1. Dispatcher is dispatching 2 window of events.
     * 2. First window consumption is successfully done.
     * 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
     *    causing the dispatcher to wait.
     * 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
     *    cause rollback in normal scenario.
     * 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
     *    calls MUST be made.
     */
void testShutdownBeforeRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
    log.setLevel(Level.INFO);
    //log.getRoot().setLevel(Level.DEBUG);
    LOG.info("start");
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 8;
    final int numEventsPerWindow = 4;
    final int payloadSize = 200;
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    for (DbusEvent e : srcTestEvents) {
        win1Size += e.size();
    }
    //serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    //Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    /**
       *
       *  Consumer with ability to wait for latch during onStartDataEventSequence()
       */
    class TimeoutDESConsumer extends TimeoutTestConsumer {

        private final CountDownLatch latch = new CountDownLatch(1);

        private int _countStartWindow = 0;

        private final int _failedRequestNumber;

        public TimeoutDESConsumer(int failedRequestNumber) {
            super(1, 1, 0, 0, 0, 0);
            _failedRequestNumber = failedRequestNumber;
        }

        public CountDownLatch getLatch() {
            return latch;
        }

        @Override
        public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
            _countStartWindow++;
            if (_countStartWindow == _failedRequestNumber) {
                // Wait for the latch to open
                try {
                    latch.await();
                } catch (InterruptedException e) {
                }
                return ConsumerCallbackResult.ERROR_FATAL;
            }
            return super.onStartDataEventSequence(startScn);
        }

        @Override
        public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
            return ConsumerCallbackResult.SUCCESS;
        }

        public int getNumBeginWindowCalls() {
            return _countStartWindow;
        }
    }
    //Create dispatcher
    //fail on second window
    final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
    100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    final Thread dispatcherThread = new Thread(dispatcher);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    // Generate RegisterRespone for schema
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    // Enqueue Necessary messages before starting dispatch
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    //comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    /*
       *  Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
       *  With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
       *  reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
       */
    readerStream.configureBlocking(false);
    //Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    //Event readers - Clients in the real world
    //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send both windows");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        // EOP events, presumably?
        Assert.assertEquals(// EOP events, presumably?
        "num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 2 == mockConsumer.getNumBeginWindowCalls();
            }
        }, "second window processing started", 5000, log);
        dispatcher.shutdown();
        // remove the barrier
        mockConsumer.getLatch().countDown();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return !dispatcherThread.isAlive();
            }
        }, "Ensure Dispatcher thread is shutdown", 5000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 0 == mockConsumer.getNumRollbacks();
            }
        }, "Ensure No Rollback is called", 10, log);
    } finally {
        reader.stop();
    }
    log.info("end\n");
}
Also used : DbusEventAppender(com.linkedin.databus.core.test.DbusEventAppender) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) StreamEventsArgs(com.linkedin.databus.core.StreamEventsArgs) List(java.util.List) ArrayList(java.util.ArrayList) Vector(java.util.Vector) DbusEventBufferReader(com.linkedin.databus.core.test.DbusEventBufferReader) ConditionCheck(com.linkedin.databus2.test.ConditionCheck) StreamEventsResult(com.linkedin.databus.core.StreamEventsResult) MultiConsumerCallback(com.linkedin.databus.client.consumer.MultiConsumerCallback) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) StreamConsumerCallbackFactory(com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) DbusEventGenerator(com.linkedin.databus.core.test.DbusEventGenerator) Pipe(java.nio.channels.Pipe) CountDownLatch(java.util.concurrent.CountDownLatch) Checkpoint(com.linkedin.databus.core.Checkpoint) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) DbusEventDecoder(com.linkedin.databus.client.pub.DbusEventDecoder) Checkpoint(com.linkedin.databus.core.Checkpoint) SelectingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer) SCN(com.linkedin.databus.client.pub.SCN) Test(org.testng.annotations.Test)

Aggregations

Checkpoint (com.linkedin.databus.core.Checkpoint)139 Test (org.testng.annotations.Test)88 ArrayList (java.util.ArrayList)46 RegisterResponseEntry (com.linkedin.databus2.core.container.request.RegisterResponseEntry)42 HashMap (java.util.HashMap)42 List (java.util.List)42 IdNamePair (com.linkedin.databus.core.util.IdNamePair)34 DefaultHttpResponse (org.jboss.netty.handler.codec.http.DefaultHttpResponse)29 ChannelBuffer (org.jboss.netty.buffer.ChannelBuffer)27 DefaultHttpChunk (org.jboss.netty.handler.codec.http.DefaultHttpChunk)25 HttpResponse (org.jboss.netty.handler.codec.http.HttpResponse)23 HttpChunk (org.jboss.netty.handler.codec.http.HttpChunk)22 BootstrapDatabaseTooOldException (com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException)20 DefaultHttpChunkTrailer (org.jboss.netty.handler.codec.http.DefaultHttpChunkTrailer)16 HttpChunkTrailer (org.jboss.netty.handler.codec.http.HttpChunkTrailer)16 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)15 IOException (java.io.IOException)15 ServerInfo (com.linkedin.databus.client.pub.ServerInfo)14 Logger (org.apache.log4j.Logger)14 InetSocketAddress (java.net.InetSocketAddress)13