use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.
the class TestGenericDispatcher method testPartialWindowRollback.
@Test(groups = { "small", "functional" })
public /**
* Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
* with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
* is correctly triggered.
*
* The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
* callback.
*
* 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
* 2) Read partial second window: e2_1 e2_2
* 3) The above should fail -- verify that rollback is called
* 4) Read the rest
*/
void testPartialWindowRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
//log.setLevel(Level.INFO);
log.info("start");
final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
//Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 9;
//1-based number of the event callback to fail
final int numOfFailureEvent = 5;
final int numEventsPerWindow = 3;
final int payloadSize = 200;
final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
int win2Size = 0;
int eventN = 0;
for (DbusEvent e : srcTestEvents) {
eventN++;
if (eventN <= numEventsPerWindow) {
win1Size += e.size();
} else if (eventN <= numOfFailureEvent) {
win2Size += e.size();
}
}
//serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
//Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
//Create dispatcher
final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
dispatcher.setSchemaIdCheck(false);
Thread dispatcherThread = new Thread(dispatcher);
dispatcherThread.setDaemon(true);
log.info("starting dispatcher thread");
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
//stream the events from the source buffer without the EOW
//comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
readerStream.configureBlocking(false);
//Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
//Event readers - Clients in the real world
//Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send first window -- that one should be OK");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == callbackStats.getNumSysEventsProcessed();
}
}, "first window processed", 5000, log);
log.info("send the second partial window -- that one should cause an error");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
log.info("wait for dispatcher to finish");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("events received: " + callbackStats.getNumDataEventsReceived());
return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
}
}, "all events until the error processed", 5000, log);
log.info("all data events have been received but no EOW");
Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
//at least one failing event therefore < numOfFailureEvent events can be processed
Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
//onDataEvent callbacks for e2_1 and e2_2 get cancelled
Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
//only one EOW
Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
log.info("Send the remainder of the window");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
//remaining events + EOWs
Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
log.info("wait for the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == mockConsumer.getNumRollbacks();
}
}, "rollback seen", 5000, log);
log.info("wait for dispatcher to finish after the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
return numWindows == callbackStats.getNumSysEventsProcessed();
}
}, "all events processed", 5000, log);
} finally {
reader.stop();
dispatcher.shutdown();
log.info("all events processed");
verifyNoLocks(null, srcEventsBuf);
verifyNoLocks(null, destEventsBuf);
}
Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
log.info("end\n");
}
use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.
the class DbusEventAppender method run.
@Override
public void run() {
//append events into buffer serially with varying window sizes;
long lastScn = -1;
_count = 0;
int dataEventCount = 0;
int bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
int maxCount = (int) (_fraction * _events.size());
for (DbusEvent ev : _events) {
if (dataEventCount >= maxCount) {
break;
}
long evScn = ev.sequence();
if (lastScn != evScn) {
//new window;
if (lastScn == -1) {
// Test DDSDBUS-1109 by skipping the start() call. The scn Index should be set for streamEvents() to work correctly
if (_invokeStartOnBuffer) {
_buffer.start(evScn - 1);
}
_buffer.startEvents();
if (_bootstrapCheckpoint != null) {
//add the initial checkpoint event to dispatcher's buffer to simulate bootstrap
addBootstrapCheckpointEventToBuffer(evScn - 1, dataEventCount, 1);
}
} else {
++_count;
if (_callInternalListeners) {
if (0 == bootstrapCheckpoints) {
_buffer.endEvents(lastScn, _stats);
} else {
addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
}
--bootstrapCheckpoints;
} else {
if (0 == bootstrapCheckpoints) {
_buffer.endEvents(true, lastScn, false, false, _stats);
} else {
//simulate bootstrap calls (snapshot)
addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
}
--bootstrapCheckpoints;
}
if (!_bufferReflector.validateBuffer()) {
throw new RuntimeException("Buffer validation 1 failed");
}
if (bootstrapCheckpoints < 0) {
_buffer.startEvents();
bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
}
}
if (!_bufferReflector.validateBuffer()) {
throw new RuntimeException("Buffer validation 2 failed");
}
lastScn = evScn;
}
dataEventCount = addEventToBuffer(ev, dataEventCount);
++_count;
}
if ((lastScn != -1) && (maxCount == _events.size())) {
++_count;
_buffer.endEvents(lastScn, _stats);
}
_dataEvents = dataEventCount;
}
use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.
the class TestGoldenGateEventProducer method testAddEventToBuffer.
@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
// No rate control
long rate = 0;
PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
long scn = 10;
DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
int sourceId = 505;
HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
Object key = new String("name");
Schema.Type keyType = Schema.Type.RECORD;
ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
keyPairs.add(kp);
Schema s = Schema.parse(avroSchema);
GenericRecord gr = new GenericData.Record(s);
gr.put("name", "phani");
DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
db.add(dbi);
TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
dbUpdates.add(dbUpdate);
long timestamp = System.nanoTime();
gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
DbusEventIterator iter = mb.acquireIterator("test");
int count = 0;
long eventTs = 0;
while (iter.hasNext()) {
DbusEvent e = iter.next();
if (count == 1) {
// first event prev control event
eventTs = e.timestampInNanos();
}
count++;
}
Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
Assert.assertEquals("Got events ", 3, count);
return;
}
use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.
the class RelayPullThread method doBootstrap.
protected void doBootstrap(ConnectionState curState) {
if (null != _lastOpenConnection) {
_lastOpenConnection.close();
_lastOpenConnection = null;
}
Checkpoint bootstrapCkpt = null;
if (_sourcesConn.getBootstrapPuller() == null) {
_log.warn("doBootstrap got called, but BootstrapPullThread is null. Is bootstrap disabled?");
return;
}
try {
bootstrapCkpt = curState.getCheckpoint().clone();
} catch (Exception e) {
String msg = "Error copying checkpoint:" + curState.getCheckpoint();
_log.error(msg, e);
BootstrapResultMessage bootstrapResultMessage = BootstrapResultMessage.createBootstrapFailedMessage(e);
doBootstrapFailed(bootstrapResultMessage);
return;
}
if (!bootstrapCkpt.isBootstrapStartScnSet()) {
bootstrapCkpt = curState.getBstCheckpointHandler().createInitialBootstrapCheckpoint(bootstrapCkpt, bootstrapCkpt.getWindowScn());
//bootstrapCkpt.setBootstrapSinceScn(Long.valueOf(bootstrapCkpt.getWindowScn()));
}
_log.info("Bootstrap begin: sinceScn=" + bootstrapCkpt.getWindowScn());
CheckpointMessage bootstrapCpMessage = CheckpointMessage.createSetCheckpointMessage(bootstrapCkpt);
_sourcesConn.getBootstrapPuller().enqueueMessage(bootstrapCpMessage);
try {
Checkpoint cpForDispatcher = new Checkpoint(bootstrapCkpt.toString());
cpForDispatcher.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
DbusEvent cpEvent = getEventFactory().createCheckpointEvent(cpForDispatcher);
writeEventToRelayDispatcher(curState, cpEvent, "Control Event to start bootstrap");
curState.switchToBootstrapRequested();
} catch (InterruptedException ie) {
_log.error("Got interrupted while writing control message to bootstrap !!", ie);
enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(ie));
} catch (Exception e) {
enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(e));
_log.error("Exception occured while switching to bootstrap: ", e);
}
//wait for bootstrap to finish
/*boolean bootstrapDone = false;
while (! bootstrapDone)
{
try
{
bootstrapPullerThread.join();
bootstrapDone = true;
}
catch (InterruptedException ie) {}
}*/
}
use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.
the class TestGenericDispatcher method runPartialWindowCheckpointPersistence.
/**
*
* @param numEvents : number of events in buffer
* @param maxWindowSize : window size expressed as number of events
* @param numFailWindow : nth end-of-window that will fail
* @throws Exception
*/
void runPartialWindowCheckpointPersistence(int numEvents, int maxWindowSize, int numFailWindow) throws Exception {
/* Experiment setup */
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
//setup consumer to fail on data callback at the nth event
int timeTakenForDataEventInMs = 1;
int timeTakenForControlEventInMs = 1;
int numFailCheckpointEvent = 0;
int numFailDataEvent = 0;
int numFailEndWindow = numFailWindow;
int numFailures = 1;
//fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
//thanks to a very small checkpoint frequency threshold
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
//Single threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
//Assumption: generates events with non-decreasing timestamps
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
double thresholdChkptPct = 5.0;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(0);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
//make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("OnlinePartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, true);
/* Launch writer */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
Thread tEmitter = new Thread(eventProducer);
//be generous ; use worst case for num control events
long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
tEmitter.start();
tEmitter.join(waitTimeMs);
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
//wait for dispatcher to finish reading the events;
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
Assert.assertFalse(tDispatcher.isAlive());
LOG.info("tConsumer: " + tConsumer);
HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
Checkpoint cp = i.getValue();
LOG.info("checkpoint=" + cp);
Assert.assertEquals(cp.getWindowOffset().longValue(), -1L);
//check if lastSeenCheckpoint by consumer is higher than scn persisted
Assert.assertTrue(tConsumer.getLastSeenCheckpointScn() > cp.getWindowScn());
//the latest event seen should be newer (or at least as new) as the checkpoint
Assert.assertTrue(tConsumer.getLastTsInNanosOfEvent() >= tConsumer.getLastTsInNanosOfWindow());
if (tConsumer.getLastSeenWindowScn() > 0) {
Assert.assertEquals(cp.getWindowScn(), tConsumer.getLastSeenWindowScn());
//check if the timestamp in checkpoint is the same as checkpoint of last completed window (ts of last event of the window)
Assert.assertEquals(tConsumer.getLastTsInNanosOfWindow(), cp.getTsNsecs());
} else {
//not even one window was processed before error; expect uninitialized timestamp
Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
}
}
}
Aggregations