Examples with DbusEventAppender com.linkedin.databus.core.test.DbusEventAppender used on opensource projects

Example 21 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestDbusEventBufferStreamEvents method testStreamLargeScn.

@Test
public /**
 * Tests streamEvents with a large scn with concurrent writes
 */
void testStreamLargeScn() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testStreamLargeScn");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(120000, 500000, 1024, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    final Checkpoint cp = Checkpoint.createOnlineConsumptionCheckpoint(1000);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final WritableByteChannel ochannel = Channels.newChannel(baos);
    final AtomicBoolean hasError = new AtomicBoolean(false);
    final AtomicInteger num = new AtomicInteger(0);
    final AtomicBoolean genComplete = new AtomicBoolean(false);
    UncaughtExceptionTrackingThread streamThread = new UncaughtExceptionTrackingThread(new Runnable() {

        @Override
        public void run() {
            try {
                while (!genComplete.get() && 0 >= num.get()) {
                    StreamEventsArgs args = new StreamEventsArgs(10000);
                    num.set(dbusBuf.streamEvents(cp, ochannel, args).getNumEventsStreamed());
                }
            } catch (ScnNotFoundException e) {
                hasError.set(true);
            } catch (OffsetNotFoundException e) {
                hasError.set(true);
            }
        }
    }, "testGetLargeScn.streamThread");
    streamThread.setDaemon(true);
    streamThread.start();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(70000, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    genComplete.set(true);
    streamThread.join(10000);
    Assert.assertFalse(streamThread.isAlive());
    Assert.assertNull(streamThread.getLastException());
    Assert.assertFalse(hasError.get());
    Assert.assertTrue(num.get() > 0);
}

Example 22 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendEventWhenLimitLessThanCapacity.

@Test
public /**
 * Test the following case:
 * A buffer (one of 3) has CWP = 222 and head at 383. Limit is 483 (capacity 500)
 * we create one event of size 211 (61 + 150 payload).
 * we call appender.run() which will add one EOP event (61) and the newly created event 211.
 * The end of event offset will be 222 + 211 + 61 = 494. It should update limit to this value.
 * because of the bug (DDSDBUS-1515) it used to fail when end of event goes beyond current limit
 * but less than the capacity
 * Now it should pass
 */
void testAppendEventWhenLimitLessThanCapacity() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventWhenLimitLessThanCapacity");
    log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(11, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    // current writing position should be 222 (id=1)
    assertEquals(0, headGenId);
    assertEquals(1, headIndexId);
    assertEquals(383, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(1, tailIndexId);
    assertEquals(222, tailOffset);
    log.info("append event to stretch beyond limit but less than capacity");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    // will add two event 61 + 150
    generator.generateEvents(1, 1, 400, 150, events);
    // Add events. this will cause limit increased
    appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    assertEquals(1, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(61, headOffset);
    assertEquals(2, tailIndexId);
    assertEquals(61, tailOffset);
    assertEquals(494, dbusBuf.getBuffer()[1].limit());
    log.info("finished");
}

Example 23 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestGenericDispatcher method testBootstrapPartialWindowScnOrdering.

@Test
public void testBootstrapPartialWindowScnOrdering() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testBootstrapPartialWindowScnOrdering");
    // log.setLevel(Level.DEBUG);
    log.info("start");
    // DDSDBUS-1889: Ensure bootstrap onCheckpoint() callback receives bootstrapSinceScn - not some scn.
    int numEvents = 100;
    int maxWindowSize = 25;
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    // setup consumer to fail on end of first full window
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = 1;
    int numFailures = 1;
    // fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    // thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    // Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    // make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    BootstrapDispatcher dispatcher = new BootstrapDispatcher("bootstrapPartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, // relaypuller
    null, // mbean server
    null, // ClientImpl
    null, // registrationId
    null, // logger
    null);
    dispatcher.setSchemaIdCheck(false);
    BootstrapCheckpointHandler cptHandler = new BootstrapCheckpointHandler("source1");
    long sinceScn = 15000L;
    long startTsNsecs = System.nanoTime();
    final Checkpoint initCheckpoint = cptHandler.createInitialBootstrapCheckpoint(null, sinceScn);
    initCheckpoint.setBootstrapStartNsecs(startTsNsecs);
    initCheckpoint.setBootstrapStartScn(0L);
    /* Launch writer */
    // numBootstrapCheckpoint - number of checkpoints before writing end of period
    int numBootstrapCheckpoint = 4;
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, numBootstrapCheckpoint, null);
    // simulate bootstrap server; use this checkpoint as init checkpoint
    eventProducer.setBootstrapCheckpoint(initCheckpoint);
    Thread tEmitter = new Thread(eventProducer);
    // be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    // expect dispatcher to fail - at end of window
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    Assert.assertTrue(cps.size() > 0);
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getConsumptionMode(), DbusClientMode.BOOTSTRAP_SNAPSHOT);
        // check if progress has been made during bootstrap
        Assert.assertTrue(cp.getSnapshotOffset() > 0);
        // these two values should be unchanged during the course of bootstrap
        Assert.assertEquals(sinceScn, cp.getBootstrapSinceScn().longValue());
        Assert.assertEquals(startTsNsecs, cp.getBootstrapStartNsecs());
        // the tsNsec normally udpdated by client at end of window should be a no-op during bootstrap
        Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        // the scn passed to consumers during onCheckpoint should be the sinceSCN and not any other interim value
        Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), tConsumer.getLastSeenCheckpointScn());
    }
    log.info("end\n");
}

Example 24 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestGenericDispatcher method runPartialWindowCheckpointPersistence.

/**
 * @param numEvents : number of events in buffer
 * @param maxWindowSize : window size expressed as number of events
 * @param numFailWindow : nth end-of-window that will fail
 * @throws Exception
 */
void runPartialWindowCheckpointPersistence(int numEvents, int maxWindowSize, int numFailWindow) throws Exception {
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    // setup consumer to fail on data callback at the nth event
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = numFailWindow;
    int numFailures = 1;
    // fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    // thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    // Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    // Assumption: generates events with  non-decreasing timestamps
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    // make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("OnlinePartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    // be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    // wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getWindowOffset().longValue(), -1L);
        // check if lastSeenCheckpoint by consumer is higher than scn persisted
        Assert.assertTrue(tConsumer.getLastSeenCheckpointScn() > cp.getWindowScn());
        // the latest event seen should be newer (or at least as new) as the checkpoint
        Assert.assertTrue(tConsumer.getLastTsInNanosOfEvent() >= tConsumer.getLastTsInNanosOfWindow());
        if (tConsumer.getLastSeenWindowScn() > 0) {
            Assert.assertEquals(cp.getWindowScn(), tConsumer.getLastSeenWindowScn());
            // check if the timestamp in checkpoint is the same as checkpoint of last completed window (ts of last event of the window)
            Assert.assertEquals(tConsumer.getLastTsInNanosOfWindow(), cp.getTsNsecs());
        } else {
            // not even one window was processed before error; expect uninitialized timestamp
            Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        }
    }
}

Example 25 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestGenericDispatcher method runAbsentSchemaTest.

void runAbsentSchemaTest(boolean setSchemaCheck) throws Exception {
    /* Experiment setup */
    int numEvents = 100;
    int maxWindowSize = 20;
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    // setup consumer to fail on data callback at the nth event
    DataDecodingConsumer tConsumer = new DataDecodingConsumer();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    // Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    // the schemaIds generated here are random. They will not be the same as those computed in the dispatcher.
    // The result is either the processing will fail early (desired behaviour) or during event decoding in the onDataEvent()
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.getDispatcherRetries().setMaxRetryNum(1);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    // make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, false);
    // DDSDBUS-3421; set schema check to true
    dispatcher.setSchemaIdCheck(setSchemaCheck);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    // be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * 1 + numEvents * 1) * 4;
    tEmitter.join(waitTimeMs);
    // wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    // asserts
    if (!setSchemaCheck) {
        // decoding fails many errors show up;
        Assert.assertTrue(tConsumer.getNumDataEvents() > 0);
        Assert.assertTrue(tConsumer.getNumErrors() > 0);
    } else {
        // never gets to decoding; but error shows up (exactly one - dispatcher retries set to 1);
        Assert.assertEquals(0, tConsumer.getNumDataEvents());
        Assert.assertEquals(1, tConsumer.getNumErrors());
    }
}