Search in sources :

Example 11 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class TestGenericDispatcher method testPartialWindowRollback.

@Test(groups = { "small", "functional" })
public /**
     * Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
     * with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
     * is correctly triggered.
     *
     * The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
     * callback.
     *
     * 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
     * 2) Read partial second window: e2_1 e2_2
     * 3) The above should fail -- verify that rollback is called
     * 4) Read the rest
     */
void testPartialWindowRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
    //log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    //Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 9;
    //1-based number of the event callback to fail
    final int numOfFailureEvent = 5;
    final int numEventsPerWindow = 3;
    final int payloadSize = 200;
    final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    int win2Size = 0;
    int eventN = 0;
    for (DbusEvent e : srcTestEvents) {
        eventN++;
        if (eventN <= numEventsPerWindow) {
            win1Size += e.size();
        } else if (eventN <= numOfFailureEvent) {
            win2Size += e.size();
        }
    }
    //serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    //Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    //Create dispatcher
    final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    dispatcher.setSchemaIdCheck(false);
    Thread dispatcherThread = new Thread(dispatcher);
    dispatcherThread.setDaemon(true);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    //stream the events from the source buffer without the EOW
    //comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    readerStream.configureBlocking(false);
    //Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    //Event readers - Clients in the real world
    //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send first window -- that one should be OK");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == callbackStats.getNumSysEventsProcessed();
            }
        }, "first window processed", 5000, log);
        log.info("send the second partial window -- that one should cause an error");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
        Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
        log.info("wait for dispatcher to finish");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("events received: " + callbackStats.getNumDataEventsReceived());
                return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
            }
        }, "all events until the error processed", 5000, log);
        log.info("all data events have been received but no EOW");
        Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
        Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
        //at least one failing event therefore < numOfFailureEvent events can be processed
        Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
        //onDataEvent callbacks for e2_1 and e2_2 get cancelled
        Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
        //only one EOW
        Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
        log.info("Send the remainder of the window");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
        //remaining events + EOWs
        Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
        log.info("wait for the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == mockConsumer.getNumRollbacks();
            }
        }, "rollback seen", 5000, log);
        log.info("wait for dispatcher to finish after the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
                return numWindows == callbackStats.getNumSysEventsProcessed();
            }
        }, "all events processed", 5000, log);
    } finally {
        reader.stop();
        dispatcher.shutdown();
        log.info("all events processed");
        verifyNoLocks(null, srcEventsBuf);
        verifyNoLocks(null, destEventsBuf);
    }
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}
Also used : DbusEventAppender(com.linkedin.databus.core.test.DbusEventAppender) DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) StreamEventsArgs(com.linkedin.databus.core.StreamEventsArgs) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) List(java.util.List) ArrayList(java.util.ArrayList) Vector(java.util.Vector) DbusEventBufferReader(com.linkedin.databus.core.test.DbusEventBufferReader) ConditionCheck(com.linkedin.databus2.test.ConditionCheck) StreamConsumerCallbackFactory(com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) StreamEventsResult(com.linkedin.databus.core.StreamEventsResult) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) MultiConsumerCallback(com.linkedin.databus.client.consumer.MultiConsumerCallback) DbusEventGenerator(com.linkedin.databus.core.test.DbusEventGenerator) Pipe(java.nio.channels.Pipe) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) Checkpoint(com.linkedin.databus.core.Checkpoint) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) Checkpoint(com.linkedin.databus.core.Checkpoint) SelectingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) Level(org.apache.log4j.Level) Test(org.testng.annotations.Test)

Example 12 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class DbusEventAppender method run.

@Override
public void run() {
    //append events into buffer serially with varying  window sizes;
    long lastScn = -1;
    _count = 0;
    int dataEventCount = 0;
    int bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
    int maxCount = (int) (_fraction * _events.size());
    for (DbusEvent ev : _events) {
        if (dataEventCount >= maxCount) {
            break;
        }
        long evScn = ev.sequence();
        if (lastScn != evScn) {
            //new window;
            if (lastScn == -1) {
                // Test DDSDBUS-1109 by skipping the start() call. The scn Index should be set for streamEvents() to work correctly
                if (_invokeStartOnBuffer) {
                    _buffer.start(evScn - 1);
                }
                _buffer.startEvents();
                if (_bootstrapCheckpoint != null) {
                    //add the initial checkpoint event to dispatcher's buffer to simulate bootstrap
                    addBootstrapCheckpointEventToBuffer(evScn - 1, dataEventCount, 1);
                }
            } else {
                ++_count;
                if (_callInternalListeners) {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(lastScn, _stats);
                    } else {
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                } else {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(true, lastScn, false, false, _stats);
                    } else {
                        //simulate bootstrap calls (snapshot)
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                }
                if (!_bufferReflector.validateBuffer()) {
                    throw new RuntimeException("Buffer validation 1 failed");
                }
                if (bootstrapCheckpoints < 0) {
                    _buffer.startEvents();
                    bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
                }
            }
            if (!_bufferReflector.validateBuffer()) {
                throw new RuntimeException("Buffer validation 2 failed");
            }
            lastScn = evScn;
        }
        dataEventCount = addEventToBuffer(ev, dataEventCount);
        ++_count;
    }
    if ((lastScn != -1) && (maxCount == _events.size())) {
        ++_count;
        _buffer.endEvents(lastScn, _stats);
    }
    _dataEvents = dataEventCount;
}
Also used : DbusEvent(com.linkedin.databus.core.DbusEvent) Checkpoint(com.linkedin.databus.core.Checkpoint)

Example 13 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}
Also used : PhysicalSourceStaticConfig(com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig) TransactionState(com.linkedin.databus2.ggParser.XmlStateMachine.TransactionState) Schema(org.apache.avro.Schema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) ArrayList(java.util.ArrayList) TransactionInfo(com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo) GenericRecord(org.apache.avro.generic.GenericRecord) GenericRecord(org.apache.avro.generic.GenericRecord) HashSet(java.util.HashSet) DbusEvent(com.linkedin.databus.core.DbusEvent) DBUpdateImage(com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage) ColumnsState(com.linkedin.databus2.ggParser.XmlStateMachine.ColumnsState) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) DbusEventIterator(com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator) Test(org.testng.annotations.Test)

Example 14 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class RelayPullThread method doBootstrap.

protected void doBootstrap(ConnectionState curState) {
    if (null != _lastOpenConnection) {
        _lastOpenConnection.close();
        _lastOpenConnection = null;
    }
    Checkpoint bootstrapCkpt = null;
    if (_sourcesConn.getBootstrapPuller() == null) {
        _log.warn("doBootstrap got called, but BootstrapPullThread is null. Is bootstrap disabled?");
        return;
    }
    try {
        bootstrapCkpt = curState.getCheckpoint().clone();
    } catch (Exception e) {
        String msg = "Error copying checkpoint:" + curState.getCheckpoint();
        _log.error(msg, e);
        BootstrapResultMessage bootstrapResultMessage = BootstrapResultMessage.createBootstrapFailedMessage(e);
        doBootstrapFailed(bootstrapResultMessage);
        return;
    }
    if (!bootstrapCkpt.isBootstrapStartScnSet()) {
        bootstrapCkpt = curState.getBstCheckpointHandler().createInitialBootstrapCheckpoint(bootstrapCkpt, bootstrapCkpt.getWindowScn());
    //bootstrapCkpt.setBootstrapSinceScn(Long.valueOf(bootstrapCkpt.getWindowScn()));
    }
    _log.info("Bootstrap begin: sinceScn=" + bootstrapCkpt.getWindowScn());
    CheckpointMessage bootstrapCpMessage = CheckpointMessage.createSetCheckpointMessage(bootstrapCkpt);
    _sourcesConn.getBootstrapPuller().enqueueMessage(bootstrapCpMessage);
    try {
        Checkpoint cpForDispatcher = new Checkpoint(bootstrapCkpt.toString());
        cpForDispatcher.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
        DbusEvent cpEvent = getEventFactory().createCheckpointEvent(cpForDispatcher);
        writeEventToRelayDispatcher(curState, cpEvent, "Control Event to start bootstrap");
        curState.switchToBootstrapRequested();
    } catch (InterruptedException ie) {
        _log.error("Got interrupted while writing control message to bootstrap !!", ie);
        enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(ie));
    } catch (Exception e) {
        enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(e));
        _log.error("Exception occured while switching to bootstrap: ", e);
    }
//wait for bootstrap to finish
/*boolean bootstrapDone = false;
    while (! bootstrapDone)
    {
      try
      {
        bootstrapPullerThread.join();
        bootstrapDone = true;
      }
      catch (InterruptedException ie) {}
    }*/
}
Also used : Checkpoint(com.linkedin.databus.core.Checkpoint) DbusEvent(com.linkedin.databus.core.DbusEvent) InvalidEventException(com.linkedin.databus.core.InvalidEventException) ScnNotFoundException(com.linkedin.databus.core.ScnNotFoundException) PendingEventTooLargeException(com.linkedin.databus.core.PendingEventTooLargeException) BootstrapDatabaseTooOldException(com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException) DatabusException(com.linkedin.databus2.core.DatabusException) PullerRetriesExhaustedException(com.linkedin.databus.core.PullerRetriesExhaustedException)

Example 15 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class TestGenericDispatcher method runPartialWindowCheckpointPersistence.

/**
     *
     * @param numEvents : number of events in buffer
     * @param maxWindowSize : window size expressed as number of events
     * @param numFailWindow : nth end-of-window that will fail
     * @throws Exception
     */
void runPartialWindowCheckpointPersistence(int numEvents, int maxWindowSize, int numFailWindow) throws Exception {
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    //setup consumer to fail on data callback at the nth event
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = numFailWindow;
    int numFailures = 1;
    //fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    //thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    //Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    //Assumption: generates events with  non-decreasing timestamps
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    //make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("OnlinePartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    //be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    //wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getWindowOffset().longValue(), -1L);
        //check if lastSeenCheckpoint by consumer is higher than scn persisted
        Assert.assertTrue(tConsumer.getLastSeenCheckpointScn() > cp.getWindowScn());
        //the latest event seen should be newer (or at least as new) as the checkpoint
        Assert.assertTrue(tConsumer.getLastTsInNanosOfEvent() >= tConsumer.getLastTsInNanosOfWindow());
        if (tConsumer.getLastSeenWindowScn() > 0) {
            Assert.assertEquals(cp.getWindowScn(), tConsumer.getLastSeenWindowScn());
            //check if the timestamp in checkpoint is the same as checkpoint of last completed window (ts of last event of the window)
            Assert.assertEquals(tConsumer.getLastTsInNanosOfWindow(), cp.getTsNsecs());
        } else {
            //not even one window was processed before error; expect uninitialized timestamp
            Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        }
    }
}
Also used : DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) DbusEventAppender(com.linkedin.databus.core.test.DbusEventAppender) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList) IdNamePair(com.linkedin.databus.core.util.IdNamePair) Vector(java.util.Vector) SelectingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer) DelegatingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.DelegatingDatabusCombinedConsumer) AbstractDatabusCombinedConsumer(com.linkedin.databus.client.consumer.AbstractDatabusCombinedConsumer) StreamConsumerCallbackFactory(com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory) DbusEvent(com.linkedin.databus.core.DbusEvent) MultiConsumerCallback(com.linkedin.databus.client.consumer.MultiConsumerCallback) DbusEventGenerator(com.linkedin.databus.core.test.DbusEventGenerator) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) Checkpoint(com.linkedin.databus.core.Checkpoint) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) Checkpoint(com.linkedin.databus.core.Checkpoint) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

DbusEvent (com.linkedin.databus.core.DbusEvent)30 Test (org.testng.annotations.Test)21 ArrayList (java.util.ArrayList)20 IdNamePair (com.linkedin.databus.core.util.IdNamePair)18 HashMap (java.util.HashMap)18 Checkpoint (com.linkedin.databus.core.Checkpoint)12 DatabusStreamConsumer (com.linkedin.databus.client.pub.DatabusStreamConsumer)11 DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)11 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)11 Hashtable (java.util.Hashtable)10 UnifiedClientStats (com.linkedin.databus.client.pub.mbean.UnifiedClientStats)9 AfterTest (org.testng.annotations.AfterTest)9 BeforeTest (org.testng.annotations.BeforeTest)9 DatabusV2ConsumerRegistration (com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration)8 MultiConsumerCallback (com.linkedin.databus.client.consumer.MultiConsumerCallback)8 StreamConsumerCallbackFactory (com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory)8 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)8 DbusEventAppender (com.linkedin.databus.core.test.DbusEventAppender)8 DbusEventGenerator (com.linkedin.databus.core.test.DbusEventGenerator)8 UncaughtExceptionTrackingThread (com.linkedin.databus.core.util.UncaughtExceptionTrackingThread)8