Examples with DatabusSubscription com.linkedin.databus.core.data_model.DatabusSubscription used on opensource projects

Example 36 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testBootstrapPartialWindowScnOrdering.

@Test
public void testBootstrapPartialWindowScnOrdering() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testBootstrapPartialWindowScnOrdering");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    //DDSDBUS-1889: Ensure bootstrap onCheckpoint() callback receives bootstrapSinceScn - not some scn.
    int numEvents = 100;
    int maxWindowSize = 25;
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    //setup consumer to fail on end of first full window
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = 1;
    int numFailures = 1;
    //fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    //thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    //Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    //make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    BootstrapDispatcher dispatcher = new BootstrapDispatcher("bootstrapPartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, //relaypuller
    null, //mbean server
    null, //ClientImpl
    null, //registrationId
    null, // logger
    null);
    dispatcher.setSchemaIdCheck(false);
    BootstrapCheckpointHandler cptHandler = new BootstrapCheckpointHandler("source1");
    long sinceScn = 15000L;
    long startTsNsecs = System.nanoTime();
    final Checkpoint initCheckpoint = cptHandler.createInitialBootstrapCheckpoint(null, sinceScn);
    initCheckpoint.setBootstrapStartNsecs(startTsNsecs);
    initCheckpoint.setBootstrapStartScn(0L);
    /* Launch writer */
    //numBootstrapCheckpoint - number of checkpoints before writing end of period
    int numBootstrapCheckpoint = 4;
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, numBootstrapCheckpoint, null);
    //simulate bootstrap server; use this checkpoint as init checkpoint
    eventProducer.setBootstrapCheckpoint(initCheckpoint);
    Thread tEmitter = new Thread(eventProducer);
    //be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    //expect dispatcher to fail - at end of window
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    Assert.assertTrue(cps.size() > 0);
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getConsumptionMode(), DbusClientMode.BOOTSTRAP_SNAPSHOT);
        //check if progress has been made during bootstrap
        Assert.assertTrue(cp.getSnapshotOffset() > 0);
        //these two values should be unchanged during the course of bootstrap
        Assert.assertEquals(sinceScn, cp.getBootstrapSinceScn().longValue());
        Assert.assertEquals(startTsNsecs, cp.getBootstrapStartNsecs());
        //the tsNsec normally udpdated by client at end of window should be a no-op during bootstrap
        Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        //the scn passed to consumers during onCheckpoint should be the sinceSCN and not any other interim value
        Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), tConsumer.getLastSeenCheckpointScn());
    }
    log.info("end\n");
}

Example 37 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testTwoWindowEventIndependentConsumersCallbackFailure.

@Test(groups = { "small", "functional" })
public void testTwoWindowEventIndependentConsumersCallbackFailure() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testTwoWindowEventIndependentConsumersCallbackFailure");
    log.info("start");
    int source1EventsNum = 4;
    int source2EventsNum = 5;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(200L);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(new DataSourceFailingStreamConsumer("source2")), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    Hashtable<Long, AtomicInteger> keyCounts2 = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts2 = new Hashtable<Short, AtomicInteger>();
    for (Long key : keyCounts.keySet()) {
        keyCounts2.put(key, new AtomicInteger(0));
    }
    for (Short srcid : srcidCounts.keySet()) {
        srcidCounts2.put(srcid, new AtomicInteger(0));
    }
    DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts2, srcidCounts2);
    SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    DatabusV2ConsumerRegistration consumerReg2 = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg, consumerReg2);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts2.get(i).intValue());
    }
    for (long i = source2EventsNum + 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assert keyCounts.get(1L + source1EventsNum).intValue() == 0 : "correct amount of callbacks for key " + i + ":" + keyCounts.get(i).intValue();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 38 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method runDispatcherRollback.

/**
     *
     * @param numEvents  number of events that will be written out in the test
     * @param maxWindowSize  size of window expressed as #events
     * @param numFailDataEvent  the nth data event at which failure occurs; 0 == no failures
     * @param numFailCheckpointEvent  the nth checkpoint event at which failure occurs; 0 == no failures
     * @param numFailEndWindow  the nth end-of-window at which failure occurs; 0 == no failures
     * @param thresholdPct  checkpointThresholdPct - forcible checkpoint before end-of-window
     * @param negativeTest  is this test supposed to fail
     * @param numFailures  number of failures expected (across all error types); in effect controls number of rollbacks
     * @param bootstrapCheckpointsPerWindow  k bootstrap checkpoint events are written for every one end-of-window event
     * @param timeTakenForDataEventInMs  time taken for processing data events
     * @param timeTakenForControlEventInMs  time taken for processing control events
     * @param wrapAround  use a smaller producer buffer so that events will wrap around
     */
protected void runDispatcherRollback(int numEvents, int maxWindowSize, int numFailDataEvent, int numFailCheckpointEvent, int numFailEndWindow, double thresholdPct, boolean negativeTest, int numFailures, int bootstrapCheckpointsPerWindow, long timeTakenForDataEventInMs, long timeTakenForControlEventInMs, boolean wrapAround) throws Exception {
    LOG.info("Running dispatcher rollback with: " + "numEvents=" + numEvents + " maxWindowSize=" + maxWindowSize + " numFailDataEvent=" + numFailDataEvent + " numFailCheckpoint=" + numFailCheckpointEvent + " numFailEndWindow=" + numFailEndWindow + " thresholdPct=" + thresholdPct + " negativeTest=" + negativeTest + " numFailures=" + numFailures + " bootstrapCheckpointsPerWindow=" + bootstrapCheckpointsPerWindow + " timeTakenForDataEventsInMs=" + timeTakenForDataEventInMs + " timeTakenForControlEventsInMs=" + timeTakenForControlEventInMs + " wrapAround=" + wrapAround);
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    // set up consumer to fail on data callback at the nth event
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    // single-threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, unifiedStats), null, unifiedStats, null, null);
    /* Generate events */
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = thresholdPct;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(10);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    // make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = wrapAround ? totalSize : totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /* Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, bootstrapCheckpointsPerWindow == 0);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, bootstrapCheckpointsPerWindow, null);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    // be generous; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.join(waitTimeMs);
    // wait for dispatcher to finish reading the events
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    System.out.println("tConsumer: " + tConsumer);
    int windowBeforeDataFail = (numFailDataEvent / maxWindowSize);
    int expectedDataFaults = numFailDataEvent == 0 ? 0 : numFailures;
    int expectedCheckPointFaults = (numFailCheckpointEvent == 0 || (expectedDataFaults != 0 && numFailCheckpointEvent == windowBeforeDataFail)) ? 0 : numFailures;
    // check if all windows were logged by dispatcher; in online case;
    if (bootstrapCheckpointsPerWindow == 0) {
        Assert.assertTrue(dispatcher.getNumCheckPoints() >= (numCheckpoints - expectedCheckPointFaults));
    }
    // Consumer prespective
    // 1 or 0 faults  injected in data callbacks; success (store) differs callback by 1
    Assert.assertEquals("Mismatch between callbacks and stored data on consumer.", expectedDataFaults, tConsumer.getDataCallbackCount() - tConsumer.getStoredDataCount());
    Assert.assertTrue(tConsumer.getStoredDataCount() >= tConsumer.getNumUniqStoredEvents());
    Assert.assertEquals("Consumer failed to store expected number of checkpoints.", dispatcher.getNumCheckPoints(), tConsumer.getStoredCheckpointCount());
    // it's converted to ConsumerCallbackResult.SKIP_CHECKPOINT and therefore not seen by client metrics.
    if (expectedCheckPointFaults == 0 || expectedDataFaults > 0 || negativeTest) {
        Assert.assertTrue("Unexpected error count in consumer metrics (" + unifiedStats.getNumConsumerErrors() + "); should be greater than or equal to numFailures (" + numFailures + ").", unifiedStats.getNumConsumerErrors() >= numFailures);
    } else {
        Assert.assertEquals("Unexpected error count in consumer metrics; checkpoint errors shouldn't count. ", // unless negativeTest ...
        0, unifiedStats.getNumConsumerErrors());
    }
    // rollback behaviour; were all events re-sent?
    if (!negativeTest) {
        Assert.assertTrue(tConsumer.getNumUniqStoredEvents() == numEvents);
    } else {
        Assert.assertTrue(tConsumer.getNumUniqStoredEvents() < numEvents);
    }
    dispatcher.shutdown();
    verifyNoLocks(null, dataEventsBuffer);
}

Example 39 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method runAbsentSchemaTest.

void runAbsentSchemaTest(boolean setSchemaCheck) throws Exception {
    /* Experiment setup */
    int numEvents = 100;
    int maxWindowSize = 20;
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    //setup consumer to fail on data callback at the nth event
    DataDecodingConsumer tConsumer = new DataDecodingConsumer();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    //Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    //the schemaIds generated here are random. They will not be the same as those computed in the dispatcher.
    //The result is either the processing will fail early (desired behaviour) or during event decoding in the onDataEvent()
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.getDispatcherRetries().setMaxRetryNum(1);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    //make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, false);
    //DDSDBUS-3421; set schema check to true
    dispatcher.setSchemaIdCheck(setSchemaCheck);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    //be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * 1 + numEvents * 1) * 4;
    tEmitter.join(waitTimeMs);
    //wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    //asserts
    if (!setSchemaCheck) {
        //decoding fails many errors show up;
        Assert.assertTrue(tConsumer.getNumDataEvents() > 0);
        Assert.assertTrue(tConsumer.getNumErrors() > 0);
    } else {
        //never gets to decoding; but error shows up (exactly one - dispatcher retries set to 1);
        Assert.assertEquals(0, tConsumer.getNumDataEvents());
        Assert.assertEquals(1, tConsumer.getNumErrors());
    }
}

Example 40 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testControlEventsRemoval.

@Test(groups = { "small", "functional" })
public void testControlEventsRemoval() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testControlEventsRemoval");
    log.info("start");
    //DDSDBUS-559
    /* Consumer creation */
    int timeTakenForEventInMs = 10;
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForEventInMs);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Source configuration */
    double thresholdChkptPct = 10.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    int numEvents = 100;
    int payloadSize = 20;
    int maxWindowSize = 1;
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, payloadSize + 62, payloadSize, srcTestEvents) > 0);
    long lastWindowScn = srcTestEvents.get(srcTestEvents.size() - 1).sequence();
    int size = 0;
    for (DbusEvent e : srcTestEvents) {
        if (e.size() > size)
            size = e.size();
    }
    //make buffer large enough to hold data
    int numWindows = (numEvents / maxWindowSize) + 1;
    int producerBufferSize = (numEvents + numWindows) * size + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    final TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("freqCkpt", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    /* write events all of which  are empty windows */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, null, 1.0, true, 0);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    long freeSpaceBefore = dataEventsBuffer.getBufferFreeSpace();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    tDispatcher.join(5000);
    LOG.warn("Free Space After=" + dataEventsBuffer.getBufferFreeSpace() + " tConsumer=" + tConsumer + " expected last window=" + lastWindowScn + " last Window = " + dataEventsBuffer.lastWrittenScn());
    Assert.assertTrue(dataEventsBuffer.lastWrittenScn() == lastWindowScn);
    Assert.assertTrue(freeSpaceBefore < dataEventsBuffer.getBufferFreeSpace());
    dispatcher.shutdown();
    verifyNoLocks(null, dataEventsBuffer);
    log.info("end\n");
}