Examples with DatabusCombinedConsumer com.linkedin.databus.client.pub.DatabusCombinedConsumer used on opensource projects

Example 26 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class TestGenericDispatcher method testControlEventsRemoval.

@Test(groups = { "small", "functional" })
public void testControlEventsRemoval() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testControlEventsRemoval");
    log.info("start");
    //DDSDBUS-559
    /* Consumer creation */
    int timeTakenForEventInMs = 10;
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForEventInMs);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Source configuration */
    double thresholdChkptPct = 10.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    int numEvents = 100;
    int payloadSize = 20;
    int maxWindowSize = 1;
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, payloadSize + 62, payloadSize, srcTestEvents) > 0);
    long lastWindowScn = srcTestEvents.get(srcTestEvents.size() - 1).sequence();
    int size = 0;
    for (DbusEvent e : srcTestEvents) {
        if (e.size() > size)
            size = e.size();
    }
    //make buffer large enough to hold data
    int numWindows = (numEvents / maxWindowSize) + 1;
    int producerBufferSize = (numEvents + numWindows) * size + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    final TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("freqCkpt", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    /* write events all of which  are empty windows */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, null, 1.0, true, 0);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    long freeSpaceBefore = dataEventsBuffer.getBufferFreeSpace();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    tDispatcher.join(5000);
    LOG.warn("Free Space After=" + dataEventsBuffer.getBufferFreeSpace() + " tConsumer=" + tConsumer + " expected last window=" + lastWindowScn + " last Window = " + dataEventsBuffer.lastWrittenScn());
    Assert.assertTrue(dataEventsBuffer.lastWrittenScn() == lastWindowScn);
    Assert.assertTrue(freeSpaceBefore < dataEventsBuffer.getBufferFreeSpace());
    dispatcher.shutdown();
    verifyNoLocks(null, dataEventsBuffer);
    log.info("end\n");
}