use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testShutdownBeforeRollback.
@Test(groups = { "small", "functional" })
public /**
*
* 1. Dispatcher is dispatching 2 window of events.
* 2. First window consumption is successfully done.
* 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
* causing the dispatcher to wait.
* 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
* cause rollback in normal scenario.
* 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
* calls MUST be made.
*/
void testShutdownBeforeRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
log.setLevel(Level.INFO);
//log.getRoot().setLevel(Level.DEBUG);
LOG.info("start");
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 8;
final int numEventsPerWindow = 4;
final int payloadSize = 200;
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
for (DbusEvent e : srcTestEvents) {
win1Size += e.size();
}
//serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
//Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
/**
*
* Consumer with ability to wait for latch during onStartDataEventSequence()
*/
class TimeoutDESConsumer extends TimeoutTestConsumer {
private final CountDownLatch latch = new CountDownLatch(1);
private int _countStartWindow = 0;
private final int _failedRequestNumber;
public TimeoutDESConsumer(int failedRequestNumber) {
super(1, 1, 0, 0, 0, 0);
_failedRequestNumber = failedRequestNumber;
}
public CountDownLatch getLatch() {
return latch;
}
@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
_countStartWindow++;
if (_countStartWindow == _failedRequestNumber) {
// Wait for the latch to open
try {
latch.await();
} catch (InterruptedException e) {
}
return ConsumerCallbackResult.ERROR_FATAL;
}
return super.onStartDataEventSequence(startScn);
}
@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
return ConsumerCallbackResult.SUCCESS;
}
public int getNumBeginWindowCalls() {
return _countStartWindow;
}
}
//Create dispatcher
//fail on second window
final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
final Thread dispatcherThread = new Thread(dispatcher);
log.info("starting dispatcher thread");
dispatcherThread.start();
// Generate RegisterRespone for schema
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
// Enqueue Necessary messages before starting dispatch
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
//comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
/*
* Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
* With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
* reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
*/
readerStream.configureBlocking(false);
//Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
//Event readers - Clients in the real world
//Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send both windows");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
// EOP events, presumably?
Assert.assertEquals(// EOP events, presumably?
"num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 2 == mockConsumer.getNumBeginWindowCalls();
}
}, "second window processing started", 5000, log);
dispatcher.shutdown();
// remove the barrier
mockConsumer.getLatch().countDown();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return !dispatcherThread.isAlive();
}
}, "Ensure Dispatcher thread is shutdown", 5000, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 0 == mockConsumer.getNumRollbacks();
}
}, "Ensure No Rollback is called", 10, log);
} finally {
reader.stop();
}
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testBootstrapPartialWindowScnOrdering.
@Test
public void testBootstrapPartialWindowScnOrdering() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testBootstrapPartialWindowScnOrdering");
//log.setLevel(Level.DEBUG);
log.info("start");
//DDSDBUS-1889: Ensure bootstrap onCheckpoint() callback receives bootstrapSinceScn - not some scn.
int numEvents = 100;
int maxWindowSize = 25;
/* Experiment setup */
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
//setup consumer to fail on end of first full window
int timeTakenForDataEventInMs = 1;
int timeTakenForControlEventInMs = 1;
int numFailCheckpointEvent = 0;
int numFailDataEvent = 0;
int numFailEndWindow = 1;
int numFailures = 1;
//fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
//thanks to a very small checkpoint frequency threshold
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
//Single threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
double thresholdChkptPct = 5.0;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(0);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
//make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
BootstrapDispatcher dispatcher = new BootstrapDispatcher("bootstrapPartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, //relaypuller
null, //mbean server
null, //ClientImpl
null, //registrationId
null, // logger
null);
dispatcher.setSchemaIdCheck(false);
BootstrapCheckpointHandler cptHandler = new BootstrapCheckpointHandler("source1");
long sinceScn = 15000L;
long startTsNsecs = System.nanoTime();
final Checkpoint initCheckpoint = cptHandler.createInitialBootstrapCheckpoint(null, sinceScn);
initCheckpoint.setBootstrapStartNsecs(startTsNsecs);
initCheckpoint.setBootstrapStartScn(0L);
/* Launch writer */
//numBootstrapCheckpoint - number of checkpoints before writing end of period
int numBootstrapCheckpoint = 4;
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, numBootstrapCheckpoint, null);
//simulate bootstrap server; use this checkpoint as init checkpoint
eventProducer.setBootstrapCheckpoint(initCheckpoint);
Thread tEmitter = new Thread(eventProducer);
//be generous ; use worst case for num control events
long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
tEmitter.start();
tEmitter.join(waitTimeMs);
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
//expect dispatcher to fail - at end of window
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
Assert.assertFalse(tDispatcher.isAlive());
LOG.info("tConsumer: " + tConsumer);
HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
Assert.assertTrue(cps.size() > 0);
for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
Checkpoint cp = i.getValue();
LOG.info("checkpoint=" + cp);
Assert.assertEquals(cp.getConsumptionMode(), DbusClientMode.BOOTSTRAP_SNAPSHOT);
//check if progress has been made during bootstrap
Assert.assertTrue(cp.getSnapshotOffset() > 0);
//these two values should be unchanged during the course of bootstrap
Assert.assertEquals(sinceScn, cp.getBootstrapSinceScn().longValue());
Assert.assertEquals(startTsNsecs, cp.getBootstrapStartNsecs());
//the tsNsec normally udpdated by client at end of window should be a no-op during bootstrap
Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
//the scn passed to consumers during onCheckpoint should be the sinceSCN and not any other interim value
Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), tConsumer.getLastSeenCheckpointScn());
}
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testTwoWindowEventIndependentConsumersCallbackFailure.
@Test(groups = { "small", "functional" })
public void testTwoWindowEventIndependentConsumersCallbackFailure() {
final Logger log = Logger.getLogger("TestGenericDispatcher.testTwoWindowEventIndependentConsumersCallbackFailure");
log.info("start");
int source1EventsNum = 4;
int source2EventsNum = 5;
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
eventsBuf.endEvents(100L);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
eventsBuf.endEvents(200L);
DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(new DataSourceFailingStreamConsumer("source2")), keyCounts, srcidCounts);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
Hashtable<Long, AtomicInteger> keyCounts2 = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts2 = new Hashtable<Short, AtomicInteger>();
for (Long key : keyCounts.keySet()) {
keyCounts2.put(key, new AtomicInteger(0));
}
for (Short srcid : srcidCounts.keySet()) {
srcidCounts2.put(srcid, new AtomicInteger(0));
}
DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts2, srcidCounts2);
SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
DatabusV2ConsumerRegistration consumerReg2 = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg, consumerReg2);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
Thread dispatcherThread = new Thread(dispatcher);
//dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
}
dispatcher.shutdown();
for (long i = 1; i <= source1EventsNum; ++i) {
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts2.get(i).intValue());
}
for (long i = source2EventsNum + 1; i <= source1EventsNum + source2EventsNum; ++i) {
assert keyCounts.get(1L + source1EventsNum).intValue() == 0 : "correct amount of callbacks for key " + i + ":" + keyCounts.get(i).intValue();
}
verifyNoLocks(null, eventsBuf);
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method runDispatcherRollback.
/**
*
* @param numEvents number of events that will be written out in the test
* @param maxWindowSize size of window expressed as #events
* @param numFailDataEvent the nth data event at which failure occurs; 0 == no failures
* @param numFailCheckpointEvent the nth checkpoint event at which failure occurs; 0 == no failures
* @param numFailEndWindow the nth end-of-window at which failure occurs; 0 == no failures
* @param thresholdPct checkpointThresholdPct - forcible checkpoint before end-of-window
* @param negativeTest is this test supposed to fail
* @param numFailures number of failures expected (across all error types); in effect controls number of rollbacks
* @param bootstrapCheckpointsPerWindow k bootstrap checkpoint events are written for every one end-of-window event
* @param timeTakenForDataEventInMs time taken for processing data events
* @param timeTakenForControlEventInMs time taken for processing control events
* @param wrapAround use a smaller producer buffer so that events will wrap around
*/
protected void runDispatcherRollback(int numEvents, int maxWindowSize, int numFailDataEvent, int numFailCheckpointEvent, int numFailEndWindow, double thresholdPct, boolean negativeTest, int numFailures, int bootstrapCheckpointsPerWindow, long timeTakenForDataEventInMs, long timeTakenForControlEventInMs, boolean wrapAround) throws Exception {
LOG.info("Running dispatcher rollback with: " + "numEvents=" + numEvents + " maxWindowSize=" + maxWindowSize + " numFailDataEvent=" + numFailDataEvent + " numFailCheckpoint=" + numFailCheckpointEvent + " numFailEndWindow=" + numFailEndWindow + " thresholdPct=" + thresholdPct + " negativeTest=" + negativeTest + " numFailures=" + numFailures + " bootstrapCheckpointsPerWindow=" + bootstrapCheckpointsPerWindow + " timeTakenForDataEventsInMs=" + timeTakenForDataEventInMs + " timeTakenForControlEventsInMs=" + timeTakenForControlEventInMs + " wrapAround=" + wrapAround);
/* Experiment setup */
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
// set up consumer to fail on data callback at the nth event
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
// single-threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, unifiedStats), null, unifiedStats, null, null);
/* Generate events */
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
double thresholdChkptPct = thresholdPct;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(10);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
// make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = wrapAround ? totalSize : totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/* Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, bootstrapCheckpointsPerWindow == 0);
/* Launch writer */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, bootstrapCheckpointsPerWindow, null);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
// be generous; use worst case for num control events
long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
tEmitter.join(waitTimeMs);
// wait for dispatcher to finish reading the events
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
System.out.println("tConsumer: " + tConsumer);
int windowBeforeDataFail = (numFailDataEvent / maxWindowSize);
int expectedDataFaults = numFailDataEvent == 0 ? 0 : numFailures;
int expectedCheckPointFaults = (numFailCheckpointEvent == 0 || (expectedDataFaults != 0 && numFailCheckpointEvent == windowBeforeDataFail)) ? 0 : numFailures;
// check if all windows were logged by dispatcher; in online case;
if (bootstrapCheckpointsPerWindow == 0) {
Assert.assertTrue(dispatcher.getNumCheckPoints() >= (numCheckpoints - expectedCheckPointFaults));
}
// Consumer prespective
// 1 or 0 faults injected in data callbacks; success (store) differs callback by 1
Assert.assertEquals("Mismatch between callbacks and stored data on consumer.", expectedDataFaults, tConsumer.getDataCallbackCount() - tConsumer.getStoredDataCount());
Assert.assertTrue(tConsumer.getStoredDataCount() >= tConsumer.getNumUniqStoredEvents());
Assert.assertEquals("Consumer failed to store expected number of checkpoints.", dispatcher.getNumCheckPoints(), tConsumer.getStoredCheckpointCount());
// it's converted to ConsumerCallbackResult.SKIP_CHECKPOINT and therefore not seen by client metrics.
if (expectedCheckPointFaults == 0 || expectedDataFaults > 0 || negativeTest) {
Assert.assertTrue("Unexpected error count in consumer metrics (" + unifiedStats.getNumConsumerErrors() + "); should be greater than or equal to numFailures (" + numFailures + ").", unifiedStats.getNumConsumerErrors() >= numFailures);
} else {
Assert.assertEquals("Unexpected error count in consumer metrics; checkpoint errors shouldn't count. ", // unless negativeTest ...
0, unifiedStats.getNumConsumerErrors());
}
// rollback behaviour; were all events re-sent?
if (!negativeTest) {
Assert.assertTrue(tConsumer.getNumUniqStoredEvents() == numEvents);
} else {
Assert.assertTrue(tConsumer.getNumUniqStoredEvents() < numEvents);
}
dispatcher.shutdown();
verifyNoLocks(null, dataEventsBuffer);
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method runAbsentSchemaTest.
void runAbsentSchemaTest(boolean setSchemaCheck) throws Exception {
/* Experiment setup */
int numEvents = 100;
int maxWindowSize = 20;
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
//setup consumer to fail on data callback at the nth event
DataDecodingConsumer tConsumer = new DataDecodingConsumer();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
//Single threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
//the schemaIds generated here are random. They will not be the same as those computed in the dispatcher.
//The result is either the processing will fail early (desired behaviour) or during event decoding in the onDataEvent()
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.getDispatcherRetries().setMaxRetryNum(1);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
//make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, false);
//DDSDBUS-3421; set schema check to true
dispatcher.setSchemaIdCheck(setSchemaCheck);
/* Launch writer */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
//be generous ; use worst case for num control events
long waitTimeMs = (numEvents * 1 + numEvents * 1) * 4;
tEmitter.join(waitTimeMs);
//wait for dispatcher to finish reading the events;
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
//asserts
if (!setSchemaCheck) {
//decoding fails many errors show up;
Assert.assertTrue(tConsumer.getNumDataEvents() > 0);
Assert.assertTrue(tConsumer.getNumErrors() > 0);
} else {
//never gets to decoding; but error shows up (exactly one - dispatcher retries set to 1);
Assert.assertEquals(0, tConsumer.getNumDataEvents());
Assert.assertEquals(1, tConsumer.getNumErrors());
}
}
Aggregations