Examples with ConditionCheck com.linkedin.databus2.test.ConditionCheck used on opensource projects

Example 31 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class MockRemoteExceptionHandler method testRelayFailOver.

@Test(groups = { "small", "functional" })
public void testRelayFailOver() throws Exception {
    final Logger log = Logger.getLogger("TestRelayPullThread.testRelayFailOver");
    log.setLevel(Level.INFO);
    log.info("-------------- start -------------------");
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.runtime.bootstrap.enabled", "false");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(2).name", "relay2");
    clientProps.setProperty("client.runtime.relay(2).port", "10002");
    clientProps.setProperty("client.runtime.relay(2).sources", "source1");
    clientProps.setProperty("client.runtime.relay(3).name", "relay3");
    clientProps.setProperty("client.runtime.relay(3).port", "10003");
    clientProps.setProperty("client.runtime.relay(3).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "9");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncFactor", "1.0");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncDelta", "1");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.initSleep", "1");
    DatabusHttpClientImpl client = new DatabusHttpClientImpl("client.", clientProps);
    Assert.assertNotNull(client, "client instantiation ok");
    final DatabusHttpClientImpl.StaticConfig clientConf = client.getClientStaticConfig();
    final DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    DbusEventBuffer.StaticConfig bufferConf = clientConf.getConnectionDefaults().getEventBuffer();
    DbusEventBuffer relayBuffer = new DbusEventBuffer(bufferConf);
    DbusEventBuffer bootstrapBuffer = new DbusEventBuffer(bufferConf);
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    Set<ServerInfo> relays = clientRtConf.getRelaysSet();
    //generate the order in which we should see the servers
    List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(relays);
    if (log.isInfoEnabled()) {
        StringBuilder sb = new StringBuilder();
        for (ServerInfo serverInfo : relayOrder) {
            sb.append(serverInfo.getName());
            sb.append(" ");
        }
        log.info("Relay order:" + sb.toString());
    }
    //This guy always fails on /sources
    final MockRelayConnection mockFailConn = new MockRelayConnection(null, null, null, serverIdx);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    //This guy succeeds on /sources but fails on /register
    final MockRelayConnection mockSuccessConn = new MockRelayConnection(sourcesResponse, null, null, serverIdx);
    DatabusRelayConnectionFactory mockConnFactory = EasyMock.createMock("mockRelayFactory", DatabusRelayConnectionFactory.class);
    //each server should be tried MAX_RETRIES time until all retries are exhausted
    for (int i = 0; i < clientConf.getConnectionDefaults().getPullerRetries().getMaxRetryNum() / 3; ++i) {
        EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockFailConn);
        EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockFailConn);
        EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn);
    }
    EasyMock.replay(mockConnFactory);
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    //Dummy connection object as expected by the puller thread
    // Note that in this case, it is ok to pass Set<relays> as all the relays serve the same source
    //"source1"
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    DatabusSourcesConnection sourcesConn = new DatabusSourcesConnection(srcConnConf, sourcesSubList, relays, null, null, null, relayBuffer, bootstrapBuffer, Executors.newCachedThreadPool(), null, null, null, null, null, null, null, mockConnFactory, null, null, null, null, new DbusEventV1Factory(), connStateFactory);
    final RelayPullThread relayPuller = new RelayPullThread("RelayPuller", sourcesConn, relayBuffer, connStateFactory, relays, new ArrayList<DbusKeyCompositeFilterConfig>(), !clientConf.getRuntime().getBootstrap().isEnabled(), clientConf.isReadLatestScnOnErrorEnabled(), clientConf.getPullerBufferUtilizationPct(), Integer.MAX_VALUE, ManagementFactory.getPlatformMBeanServer(), new DbusEventV1Factory(), null);
    RemoteExceptionHandler mockRemoteExceptionHandler = new MockRemoteExceptionHandler(sourcesConn, relayBuffer, relayPuller);
    Field field = relayPuller.getClass().getDeclaredField("_remoteExceptionHandler");
    field.setAccessible(true);
    field.set(relayPuller, mockRemoteExceptionHandler);
    mockFailConn.setCallback(relayPuller);
    mockSuccessConn.setCallback(relayPuller);
    //Let the show begin
    Thread relayPullerThread = new Thread(relayPuller);
    relayPullerThread.setDaemon(true);
    relayPullerThread.start();
    relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return mockFailConn.getSourcesCallCounter() == 6;
        }
    }, "failConn: correct number of /sources", 500, log);
    Assert.assertEquals(mockFailConn.getRegisterCallCounter(), 0, "failConn: correct number of /register");
    Assert.assertEquals(mockFailConn.getStreamCallCounter(), 0, "failConn: correct number of /stream");
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return mockSuccessConn.getSourcesCallCounter() == 3;
        }
    }, "successConn: correct number of /sources", 500, log);
    Assert.assertEquals(mockSuccessConn.getRegisterCallCounter(), 3, "successConn: correct number of /register");
    Assert.assertEquals(mockSuccessConn.getStreamCallCounter(), 0, "successConn: correct number of /stream");
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return relayPuller.getComponentStatus().getStatus() == DatabusComponentStatus.Status.SUSPENDED_ON_ERROR;
        }
    }, "puller suspended because of out of retries", 500, log);
    EasyMock.verify(mockConnFactory);
    relayPuller.enqueueMessage(LifecycleMessage.createShutdownMessage());
    relayPuller.awaitShutdown();
    log.info("------------ done --------------");
}

Example 32 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class MockRemoteExceptionHandler method testRelayPendingEvent.

@Test(groups = { "small", "functional" })
public void testRelayPendingEvent() throws Exception {
    final Logger log = Logger.getLogger("TestRelayPullThread.testRelayPendingEvent");
    log.setLevel(Level.INFO);
    log.info("-------------- start -------------------");
    //set a clientReadBufferSize equal to maxSize ; this is also used to generate an event larger than what can fit in buffer
    final int clientReadBufferSize = 100000;
    //set initReadBufferSize to a value greater than default averageEventSize which is 20K but lesser than 0.25*maxSize.
    //See DbsuEventBuffer.Config.DEFAULT_AVERAGE_EVENT_SIZE
    final int averageEventSize = 22500;
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.runtime.bootstrap.enabled", "false");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", Integer.toString(clientReadBufferSize));
    //this setting should have no effect
    clientProps.setProperty("client.connectionDefaults.eventBuffer.readBufferSize", Integer.toString(clientReadBufferSize));
    //readBufferSize init size will be set to this value
    clientProps.setProperty("client.connectionDefaults.eventBuffer.averageEventSize", Integer.toString(averageEventSize));
    //set a high value of freeBufferThreshold which is unsafe; this setting should fail
    clientProps.setProperty("client.connectionDefaults.freeBufferThreshold", "100000");
    DatabusHttpClientImpl client = new DatabusHttpClientImpl("client.", clientProps);
    Assert.assertNotNull(client, "client instantiation ok");
    final DatabusHttpClientImpl.StaticConfig clientConf = client.getClientStaticConfig();
    final DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    DbusEventBuffer.StaticConfig bufferConf = clientConf.getConnectionDefaults().getEventBuffer();
    log.info("Buffer size=" + bufferConf.getMaxSize() + " readBufferSize=" + bufferConf.getReadBufferSize() + " maxEventSize=" + bufferConf.getMaxEventSize());
    Assert.assertEquals(bufferConf.getReadBufferSize(), averageEventSize);
    //This should be equal to the maxEventSize if defaults are used for maxEventSize
    Assert.assertTrue(bufferConf.getReadBufferSize() < bufferConf.getMaxEventSize());
    Assert.assertEquals(clientConf.getConnectionDefaults().getFreeBufferThreshold(), clientConf.getConnectionDefaults().getFreeBufferThreshold());
    DbusEventBuffer relayBuffer = new DbusEventBuffer(bufferConf);
    AtomicInteger serverIdx = new AtomicInteger(-1);
    // just one relay defined in our clientProps
    Set<ServerInfo> relays = clientRtConf.getRelaysSet();
    List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(relays);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    RegisterResponseEntry rre1 = new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString());
    final HashMap<Long, List<RegisterResponseEntry>> registerResponseSources = // mapping of sourceId to list of schema versions for that ID
    new HashMap<Long, List<RegisterResponseEntry>>();
    registerResponseSources.put(1L, Arrays.asList(rre1));
    ChunkedBodyReadableByteChannel channel = new MockChunkedBodyReadableByteChannel(clientReadBufferSize + 10);
    final MockRelayConnection mockConn = new MockRelayConnection(sourcesResponse, registerResponseSources, channel, serverIdx);
    DatabusRelayConnectionFactory mockConnFactory = EasyMock.createMock("mockRelayFactory", DatabusRelayConnectionFactory.class);
    EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockConn).times(1);
    EasyMock.replay(mockConnFactory);
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    //Dummy connection object as expected by the puller thread
    // Note that in this case, it is ok to pass Set<relays> as all the relays serve the same source
    //"source1"
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    DatabusSourcesConnection sourcesConn = new DatabusSourcesConnection(srcConnConf, sourcesSubList, relays, null, null, null, relayBuffer, null, Executors.newCachedThreadPool(), null, null, null, null, null, null, null, mockConnFactory, null, null, null, null, new DbusEventV1Factory(), connStateFactory);
    final RelayPullThread relayPuller = new RelayPullThread("RelayPuller", sourcesConn, relayBuffer, connStateFactory, relays, new ArrayList<DbusKeyCompositeFilterConfig>(), !clientConf.getRuntime().getBootstrap().isEnabled(), clientConf.isReadLatestScnOnErrorEnabled(), clientConf.getPullerBufferUtilizationPct(), Integer.MAX_VALUE, ManagementFactory.getPlatformMBeanServer(), new DbusEventV1Factory(), null);
    final MockRemoteExceptionHandler mockRemoteExceptionHandler = new MockRemoteExceptionHandler(sourcesConn, relayBuffer, relayPuller);
    Field field = relayPuller.getClass().getDeclaredField("_remoteExceptionHandler");
    field.setAccessible(true);
    field.set(relayPuller, mockRemoteExceptionHandler);
    mockConn.setCallback(relayPuller);
    //Let the show begin
    Thread relayPullerThread = new Thread(relayPuller);
    relayPullerThread.setDaemon(true);
    relayPullerThread.start();
    relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
    // We will try the same relay 3 times and then our remote exception handler will be invoked.
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return relayPuller.getComponentStatus().getStatus() == Status.SUSPENDED_ON_ERROR;
        }
    }, "Reached SUSPEND_ON_ERROR state ", 500, log);
    EasyMock.verify(mockConnFactory);
    relayPuller.enqueueMessage(LifecycleMessage.createShutdownMessage());
    relayPuller.awaitShutdown();
    log.info("------------ done --------------");
}

Example 33 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestGenericDispatcher method testOneWindowTwoIndependentConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void testOneWindowTwoIndependentConsumersHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowTwoIndependentConsumersHappyPath");
    log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    //Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    final int source1EventsNum = 2;
    final int source2EventsNum = 2;
    final Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    final Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    Hashtable<Long, AtomicInteger> keyCounts2 = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts2 = new Hashtable<Short, AtomicInteger>();
    for (Long key : keyCounts.keySet()) {
        keyCounts2.put(key, new AtomicInteger(0));
    }
    for (Short srcid : srcidCounts.keySet()) {
        srcidCounts2.put(srcid, new AtomicInteger(0));
    }
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts2, srcidCounts2);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    DatabusV2ConsumerRegistration consumer2Reg = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg, consumer2Reg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    dispatcherThread.setDaemon(true);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return null != dispatcher.getDispatcherState().getEventsIterator() && !dispatcher.getDispatcherState().getEventsIterator().hasNext();
        }
    }, "all events processed", 5000, log);
    dispatcher.shutdown();
    log.info("all events processed");
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts2.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts2.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts2.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}

Example 34 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestGenericDispatcher method testShutdownBeforeRollback.

@Test(groups = { "small", "functional" })
public /**
     *
     * 1. Dispatcher is dispatching 2 window of events.
     * 2. First window consumption is successfully done.
     * 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
     *    causing the dispatcher to wait.
     * 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
     *    cause rollback in normal scenario.
     * 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
     *    calls MUST be made.
     */
void testShutdownBeforeRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
    log.setLevel(Level.INFO);
    //log.getRoot().setLevel(Level.DEBUG);
    LOG.info("start");
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 8;
    final int numEventsPerWindow = 4;
    final int payloadSize = 200;
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    for (DbusEvent e : srcTestEvents) {
        win1Size += e.size();
    }
    //serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    //Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    /**
       *
       *  Consumer with ability to wait for latch during onStartDataEventSequence()
       */
    class TimeoutDESConsumer extends TimeoutTestConsumer {

        private final CountDownLatch latch = new CountDownLatch(1);

        private int _countStartWindow = 0;

        private final int _failedRequestNumber;

        public TimeoutDESConsumer(int failedRequestNumber) {
            super(1, 1, 0, 0, 0, 0);
            _failedRequestNumber = failedRequestNumber;
        }

        public CountDownLatch getLatch() {
            return latch;
        }

        @Override
        public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
            _countStartWindow++;
            if (_countStartWindow == _failedRequestNumber) {
                // Wait for the latch to open
                try {
                    latch.await();
                } catch (InterruptedException e) {
                }
                return ConsumerCallbackResult.ERROR_FATAL;
            }
            return super.onStartDataEventSequence(startScn);
        }

        @Override
        public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
            return ConsumerCallbackResult.SUCCESS;
        }

        public int getNumBeginWindowCalls() {
            return _countStartWindow;
        }
    }
    //Create dispatcher
    //fail on second window
    final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
    100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    final Thread dispatcherThread = new Thread(dispatcher);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    // Generate RegisterRespone for schema
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    // Enqueue Necessary messages before starting dispatch
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    //comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    /*
       *  Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
       *  With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
       *  reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
       */
    readerStream.configureBlocking(false);
    //Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    //Event readers - Clients in the real world
    //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send both windows");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        // EOP events, presumably?
        Assert.assertEquals(// EOP events, presumably?
        "num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 2 == mockConsumer.getNumBeginWindowCalls();
            }
        }, "second window processing started", 5000, log);
        dispatcher.shutdown();
        // remove the barrier
        mockConsumer.getLatch().countDown();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return !dispatcherThread.isAlive();
            }
        }, "Ensure Dispatcher thread is shutdown", 5000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 0 == mockConsumer.getNumRollbacks();
            }
        }, "Ensure No Rollback is called", 10, log);
    } finally {
        reader.stop();
    }
    log.info("end\n");
}

Example 35 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow1.

/**
   * Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
   *
   * <pre>
   * R1 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^      ^        ^         ^         ^       ^       ^      ^     ^        ^
   *                   0      10       20        30        40      50      60     70    80       90
   *
   * R2 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^               ^                   ^               ^            ^
   *                   0               20                  40              60           80
   *
   * R3 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^                        ^                          ^                    ^
   *                   0                        30                         60                   90
   * </pre>
   *
   * Switch from R1 to R2 happens when windowScn = 10 and windowOffset = 8th event.
   * Switch from R2 to R3 happens when windowScn = 40 and windowOffset = 3rd event.
   *
   * @throws Exception
   */
@Test
public void testRelayFailoverPartialWindow1() throws Exception {
    final boolean debugOn = false;
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow1");
    log.setLevel(Level.INFO);
    final int eventsNum = 200;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    final long timeoutMult = debugOn ? 100000 : 1;
    log.info("simulate relay buffers");
    DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
    List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
    List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
    for (int i = 0; i < RELAY_PORT.length; ++i) {
        relayBuffer[i] = new DbusEventBuffer(_bufCfg);
        relayBuffer[i].start(0);
        WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (i + 1) * 10);
        List<Integer> ofs = wrRes.getOffsets();
        eventOfs.add(ofs);
        eventKeys.add(wrRes.getKeys());
    }
    List<DbusEventKey> key = eventKeys.get(0);
    for (int i = 1; i < RELAY_PORT.length; ++i) {
        assertEquals(" For Size index : " + i, key.size(), eventKeys.get(i).size());
        assertEquals(" For index : " + i, key, eventKeys.get(i));
        key = eventKeys.get(i);
    }
    int resp1EnfOfs = eventOfs.get(0).get(8);
    log.info("figure out an event offset inside a window:" + resp1EnfOfs);
    log.info("create client");
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    client.start();
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        log.info("figure out the connection to the relay");
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        SimpleTestServerConnection relay = null;
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("send back the /register response");
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        log.info("process /stream call and return a partial window");
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        log.info("send back the /stream response");
        final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
        HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.info("make sure the client processes the /stream response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 10;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(-1, consumer.getRollbackScn());
        int rollbackNum = 0;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
        List<Long> expSeqs = new ArrayList<Long>();
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(10L);
        long numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        log.info("now kill the relay and wait for a failover");
        serverChannel.close();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
        }, "client disconnected", 200, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        log.info("/////////// FAKING CONNECTION TO NEW RELAY //////////////");
        final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to new relay", 200, log);
        log.info("figure out the connection to the relay");
        clientChannel = newRelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("new relay selected: " + relayPort);
        relay = null;
        int relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("send back the /register response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the /register response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        log.info("process /stream call and return a partial window");
        Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        String cpString = streamMatcher.group(1);
        objCapture.clear();
        int respStartOfs = eventOfs.get(1).get(1);
        int respEndOfs = eventOfs.get(1).get(34);
        cp = new Checkpoint(cpString);
        //last window read was partial
        assertTrue(cp.getWindowOffset() > 0);
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 11000, log);
        assertEquals(20, consumer.getRollbackScn());
        log.info("one more onStartDataEventSequence because of the rolback");
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 20) * 20 + 20L);
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        ///////////////////////////////////
        //simulate a timeout on the server; the client would have sent a /stream call and there
        //will be no response from the server, so eventually it should time out and switch servers
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("Channel :" + newRelayConnInsp.getChannel());
                return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
        }, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to third relay", 200, log);
        log.info("figure out the connection to the relay");
        clientChannel = new2RelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("third relay selected: " + relayPort);
        relay = null;
        relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("SEND BACK THE /register RESPONSE");
        //clientConn.getRelayDispatcher().getLog().setLevel(Level.DEBUG);
        //RangeBasedReaderWriterLock.LOG.setLevel(Level.DEBUG);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", timeoutMult * 100, log);
        log.info("PROCESS the /stream CALL AND RETURN A PARTIAL WINDOW");
        streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        cpString = streamMatcher.group(1);
        objCapture.clear();
        respStartOfs = eventOfs.get(2).get(1);
        respEndOfs = eventOfs.get(2).get(84);
        log.debug("Checkpoint String is :" + cpString);
        cp = new Checkpoint(cpString);
        log.info("last window read was partial. So the client would have reset the windowOffset");
        assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
        assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.debug("NumEvents already seen :" + numEvents);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn() + ", NumEvents :" + stats.getTotalStats().getNumDataEvents());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
        }, "client receives /stream response, Sequences :" + consumer.getSequences(), timeoutMult * 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", timeoutMult * 1100, log);
        log.info("one more onStartDataEventSequence because of the rollback");
        assertEquals(30, consumer.getRollbackScn());
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(2).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 30) * 30 + 30L);
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
    } finally {
        client.shutdown();
    }
}