Examples with ConditionCheck com.linkedin.databus2.test.ConditionCheck used on opensource projects

Example 11 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testInStreamTimeOut3.

/**
     * same as above, but server doesn't send any data, and WriteComplete comes between WriteTimeout
     * and channel close
     * @throws Exception
     */
@Test
public void testInStreamTimeOut3() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testInStreamTimeout3");
    Level debugLevel = Level.DEBUG;
    log.setLevel(debugLevel);
    //Logger.getRootLogger().setLevel(Level.DEBUG);
    MockServerChannelHandler.LOG.setLevel(debugLevel);
    final int eventsNum = 20;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    //simulate relay buffers
    DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
    relayBuffer.start(0);
    writeEventsToBuffer(relayBuffer, eventInfos, 4);
    //prepare stream response ??????????????//
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
    // create ChunnelBuffer and fill it with events from relayBuffer
    ChannelBuffer streamResPrefix = NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
    //create client
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    // connect to a relay created in SetupClass (one out of three)
    client.start();
    // wait until a connection made
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        //get the connection
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        // figure out connection details
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        relayConnInsp.getHandler().getLog().setLevel(debugLevel);
        // wait until client is connected
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        //figure out which port we got connected to on the server side
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        // add our handler to the client's pipeline which will generate the timeout
        MockServerChannelHandler mock = new MockServerChannelHandler();
        clientChannel.getPipeline().addBefore("inflater", "mockServer", mock);
        // verify it is there
        Map<String, ChannelHandler> map = clientChannel.getPipeline().toMap();
        boolean handlerFound = false;
        for (Map.Entry<String, ChannelHandler> m : map.entrySet()) {
            if (LOG.isDebugEnabled())
                LOG.debug(m.getKey() + "=>" + m.getValue());
            if (m.getKey().equals("mockServer"))
                handlerFound = true;
        }
        Assert.assertTrue(handlerFound, "handler added");
        SimpleTestServerConnection relay = null;
        // Find the relay's object
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        HttpResponse httpResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        httpResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        httpResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, httpResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.debug("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        String msgHistory = clientConn.getRelayPullThread().getMessageHistoryLog();
        log.debug("MSG HISTORY before: " + msgHistory);
        // make sure our handler will save the 'future' of the next write operation - 'stream'
        mock.enableSaveTheFuture(true);
        // delay write complete. insert Timeout exception before that
        mock.delayWriteComplete(true);
        log.debug("send back the /register response");
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, httpResp, body);
        log.debug("make sure the client processes the response /register correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        LOG.info("*************>Message state after write complete is " + relayConnInsp.getResponseHandlerMessageState().toString());
        msgHistory = clientConn.getRelayPullThread().getMessageHistoryLog();
        log.debug("MSG HISTORY after: " + msgHistory);
        //should be one error only
        Assert.assertEquals(countOccurencesOfWord(msgHistory, "_ERROR"), 1);
    //////////////////////////////////////////////////////////////////////////////////////////////////
    /*
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return client._relayConnections.size() == 1;
            }
          }, "sources connection present", 100, log);

          //get the connection
          final DatabusSourcesConnection clientConn1 = client._relayConnections.get(0);
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != clientConn1.getRelayPullThread().getLastOpenConnection();
            }
          }, "relay connection1 present", 100, log);

          // figure out connection details
          final NettyHttpDatabusRelayConnection relayConn1 =
              (NettyHttpDatabusRelayConnection)clientConn1.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector relayConnInsp1 =
              new NettyHttpDatabusRelayConnectionInspector(relayConn1);
          relayConnInsp1.getHandler().getLog().setLevel(debugLevel);

          // wait until client is connected
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != relayConnInsp1.getChannel() && relayConnInsp1.getChannel().isConnected();
            }
          }, "client connected", 200, log);

          //figure out which port we got connected to on the server side
          Channel clientChannel1 = relayConnInsp1.getChannel();
          InetSocketAddress relayAddr1 = (InetSocketAddress)clientChannel1.getRemoteAddress();
          relayPort = relayAddr1.getPort();
          log.info("relay selected: " + relayPort);


          // do it again - no errors
          //  process the /sources request
          captureAndReplySourcesRequest(objCapture, relay, clientAddr, clientConn1, log);

          captureAndReplyRegisterRequest(objCapture, relay, clientAddr, clientConn1, log);

          log.debug("process /stream call and return a response");
          captureAndReplyStreamRequest(objCapture, relay, clientAddr, clientConn1, streamResPrefix, log);


          LOG.info("*************>Message state after write complete is " + relayConnInsp1.getResponseHandlerMessageState().toString());
          msgHistory = clientConn1.getRelayPullThread().getMessageHistoryLog();
          log.debug("MSG HISTORY after: " + msgHistory);
          Assert.assertEquals(countOccurencesOfWord(msgHistory, "_ERROR"), 1); //should be one error only



          // make sure close channel event and future failure are propagated
          TestUtil.sleep(3000);
          // get the history and validate it
          String expectedHistory =  "[START, PICK_SERVER, REQUEST_SOURCES, SOURCES_RESPONSE_SUCCESS, REQUEST_REGISTER, REGISTER_RESPONSE_SUCCESS, REQUEST_STREAM, STREAM_REQUEST_SUCCESS, STREAM_RESPONSE_DONE, REQUEST_STREAM, STREAM_REQUEST_ERROR, PICK_SERVER, REQUEST_SOURCES]".trim();
          msgHistory = clientConn.getRelayPullThread().getMessageHistoryLog().trim();
          LOG.info("MSG HISTORY: " + msgHistory);
          Assert.assertEquals(msgHistory, expectedHistory, "Puller thread message history doesn't match");
*/
    } finally {
        client.shutdown();
    }
}

Example 12 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method captureAndReplySourcesRequest.

private void captureAndReplySourcesRequest(SimpleObjectCaptureHandler objCapture, SimpleTestServerConnection relay, SocketAddress clientAddr, final DatabusSourcesConnection clientConn, final Logger log) {
    NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
    objCapture.clear();
    //send back the /sources response
    HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
    sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
    HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
    NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
    //make sure the client processes the response correctly
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
            return "1".equals(idListString);
        }
    }, "client processes /sources response", 100, log);
}

Example 13 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow2.

@Test
public /**
	 * R1 EVB Boundaries ------------------------------------------------------------------------------
	 *                   ^                        ^                          ^                    ^
	 *                   0                        30                         60                   90
	 * R2 EVB Boundaries --------------------------------------------------------------------------------
	 *                   ^               ^                   ^               ^            ^
	 *                   0               20                  40              60           80
	 *
	 * R3 EVB Boundaries --------------------------------------------------------------------------------
	 *                   ^      ^        ^         ^         ^       ^       ^      ^     ^        ^
	 *                   0      10       20        30        40      50      60     70    80       90
	 *
	 *
     *  Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
	 *
	 *  Switch from R1 -> R2 happens when windowScn = 30 and WindowOffset = 8th event
	 *  Switch from R2 -> R3 happens when windowScn = 40 and windowOffset = 5th event
	 *
	 * @throws Exception
	 */
void testRelayFailoverPartialWindow2() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow2");
    //log.setLevel(Level.DEBUG);
    final int eventsNum = 200;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    //simulate relay buffers
    DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
    List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
    List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
    for (int i = 0; i < RELAY_PORT.length; ++i) {
        relayBuffer[i] = new DbusEventBuffer(_bufCfg);
        relayBuffer[i].start(0);
        WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (RELAY_PORT.length - i) * 10);
        List<Integer> ofs = wrRes.getOffsets();
        eventOfs.add(ofs);
        eventKeys.add(wrRes.getKeys());
    }
    List<DbusEventKey> key = eventKeys.get(0);
    for (int i = 1; i < RELAY_PORT.length; ++i) {
        assertEquals(key, eventKeys.get(i));
        key = eventKeys.get(i);
    }
    //figure out an event offset inside a window
    int resp1EnfOfs = eventOfs.get(0).get(8);
    //create client
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    client.start();
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        //figure out the connection to the relay
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        SimpleTestServerConnection relay = null;
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        //process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        //send back the /register response
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        //process /stream call and return a partial window
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        //send back the /stream response
        final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
        HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 30;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.error("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(-1, consumer.getRollbackScn());
        int rollbackNum = 0;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
        List<Long> expSeqs = new ArrayList<Long>();
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(30L);
        long numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        //now kill the relay and wait for a failover
        serverChannel.close();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
        }, "client disconnected", 200, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        /////////// FAKING CONNECTION TO NEW RELAY
        final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to new relay", 200, log);
        //figure out the connection to the relay
        clientChannel = newRelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("new relay selected: " + relayPort);
        relay = null;
        int relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        //process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        //send back the /register response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        //process /stream call and return a partial window
        Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        String cpString = streamMatcher.group(1);
        objCapture.clear();
        int respStartOfs = eventOfs.get(1).get(1);
        int respEndOfs = eventOfs.get(1).get(26);
        cp = new Checkpoint(cpString);
        //last window read was partial
        assertTrue(cp.getWindowOffset() > 0);
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 11000, log);
        assertEquals(20, consumer.getRollbackScn());
        //add one more onStartDataEventsSequence after the rollback()
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++) expSeqs.add((i / 20) * 20 + 20L);
        log.info("Expected NumKeys :" + expKeys.size() + ", Got NumKeys :" + consumer.getKeys().size());
        for (int i = 0; i < expKeys.size(); i++) {
            if (!consumer.getKeys().contains(expKeys.get(i)))
                log.error(i + " Key :" + expKeys.get(i) + " missing !!");
            else
                log.info(i + " Key :" + expKeys.get(i) + " present !!");
        }
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        ///////////////////////////////////
        //simulate a timeout on the server; the client would have sent a /stream call and there
        //will be no response from the server, so eventually it should time out and switch servers
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("Channel :" + newRelayConnInsp.getChannel());
                return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
        }, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to third relay", 200, log);
        //figure out the connection to the relay
        clientChannel = new2RelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("third relay selected: " + relayPort);
        relay = null;
        relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        //process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        //send back the /register response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /sources response", 100, log);
        //process /stream call and return a partial window
        streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        cpString = streamMatcher.group(1);
        objCapture.clear();
        respStartOfs = eventOfs.get(2).get(20);
        respEndOfs = eventOfs.get(2).get(89);
        log.debug("Checkpoint String is :" + cpString);
        cp = new Checkpoint(cpString);
        //last window read was partial. So the client would have reset the windowOffset
        assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
        assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
        }, "client receives /stream response, Sequence :" + consumer.getSequences(), 10100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(30, consumer.getRollbackScn());
        //add one more onStartDataEventsSequence because of the rollback
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(2).subList(20, (int) (stats.getTotalStats().getNumDataEvents() - numEvents + 20)));
        for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++) expSeqs.add((i / 10) * 10 + 30L);
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences Size ", expSeqs.size(), consumer.getSequences().size());
        for (int i = 0; i < expSeqs.size(); i++) {
            System.out.println(i + " Exp : " + expSeqs.get(i) + ", Got :" + consumer.getSequences().get(i));
            if (expSeqs.get(i) != consumer.getSequences().get(i))
                throw new RuntimeException("Failed at " + i);
        }
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        // Expect none of the keys streamed to be missed.
        expKeys.clear();
        cp = clientConn.getRelayPullThread().getConnectionState().getCheckpoint();
        int offset = (int) ((cp.getWindowOffset() < 0) ? 0 : cp.getWindowOffset());
        LOG.info("Client ckpt is :" + cp);
        expKeys.addAll(eventKeys.get(2).subList(0, 80 + offset));
        for (int i = 0; i < expKeys.size(); i++) {
            if (!consumer.getKeys().contains(expKeys.get(i))) {
                log.error(i + " Key :" + expKeys.get(i) + " missing !!");
                throw new RuntimeException("Failed at " + i);
            } else
                log.info(i + " Key :" + expKeys.get(i) + " present !!");
        }
    } finally {
        client.shutdown();
    }
}

Example 14 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestGenericDispatcher method testPartialWindowRollback.

@Test(groups = { "small", "functional" })
public /**
     * Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
     * with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
     * is correctly triggered.
     *
     * The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
     * callback.
     *
     * 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
     * 2) Read partial second window: e2_1 e2_2
     * 3) The above should fail -- verify that rollback is called
     * 4) Read the rest
     */
void testPartialWindowRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
    //log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    //Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 9;
    //1-based number of the event callback to fail
    final int numOfFailureEvent = 5;
    final int numEventsPerWindow = 3;
    final int payloadSize = 200;
    final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    int win2Size = 0;
    int eventN = 0;
    for (DbusEvent e : srcTestEvents) {
        eventN++;
        if (eventN <= numEventsPerWindow) {
            win1Size += e.size();
        } else if (eventN <= numOfFailureEvent) {
            win2Size += e.size();
        }
    }
    //serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    //Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    //Create dispatcher
    final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    dispatcher.setSchemaIdCheck(false);
    Thread dispatcherThread = new Thread(dispatcher);
    dispatcherThread.setDaemon(true);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    //stream the events from the source buffer without the EOW
    //comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    readerStream.configureBlocking(false);
    //Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    //Event readers - Clients in the real world
    //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send first window -- that one should be OK");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == callbackStats.getNumSysEventsProcessed();
            }
        }, "first window processed", 5000, log);
        log.info("send the second partial window -- that one should cause an error");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
        Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
        log.info("wait for dispatcher to finish");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("events received: " + callbackStats.getNumDataEventsReceived());
                return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
            }
        }, "all events until the error processed", 5000, log);
        log.info("all data events have been received but no EOW");
        Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
        Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
        //at least one failing event therefore < numOfFailureEvent events can be processed
        Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
        //onDataEvent callbacks for e2_1 and e2_2 get cancelled
        Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
        //only one EOW
        Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
        log.info("Send the remainder of the window");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
        //remaining events + EOWs
        Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
        log.info("wait for the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == mockConsumer.getNumRollbacks();
            }
        }, "rollback seen", 5000, log);
        log.info("wait for dispatcher to finish after the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
                return numWindows == callbackStats.getNumSysEventsProcessed();
            }
        }, "all events processed", 5000, log);
    } finally {
        reader.stop();
        dispatcher.shutdown();
        log.info("all events processed");
        verifyNoLocks(null, srcEventsBuf);
        verifyNoLocks(null, destEventsBuf);
    }
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}

Example 15 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestGenericDispatcher method testRollbackFailure.

@Test(groups = { "small", "functional" })
public /**
     * Test a relatively big window which forces a checkpoint which fails. Then trigger
     * a rollback. The dispatcher should shutdown. */
void testRollbackFailure() throws InvalidConfigException {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testRollbackFailure");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 100, (short) 1, null, null);
    eventsBuf.endEvents(100L, null);
    RollbackFailingConsumer mockConsumer = new RollbackFailingConsumer(new StateVerifyingStreamConsumer(null), LOG, 2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(mockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DatabusSourcesConnection.Config connCfgBuilder = new DatabusSourcesConnection.Config();
    connCfgBuilder.setConsumerParallelism(1);
    connCfgBuilder.setEventBuffer(_generic100KBufferConfig);
    connCfgBuilder.setFreeBufferThreshold(10000);
    connCfgBuilder.setConsumerTimeBudgetMs(1000);
    connCfgBuilder.setCheckpointThresholdPct(5.0);
    connCfgBuilder.getDispatcherRetries().setMaxRetryNum(10);
    DatabusSourcesConnection.StaticConfig connStaticCfg = connCfgBuilder.build();
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", connStaticCfg, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                LOG.info(dispatcher.getStatus().getStatus().toString());
                LOG.info(String.valueOf(dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN));
                return dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN;
            }
        }, "dispatcher shtudown", 50000, LOG);
    } finally {
        dispatcher.shutdown();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}