use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.
the class TestGenericDispatcher method runDispatcherRollback.
/**
*
* @param numEvents number of events that will be written out in the test
* @param maxWindowSize size of window expressed as #events
* @param numFailDataEvent the nth data event at which failure occurs; 0 == no failures
* @param numFailCheckpointEvent the nth checkpoint event at which failure occurs; 0 == no failures
* @param numFailEndWindow the nth end-of-window at which failure occurs; 0 == no failures
* @param thresholdPct checkpointThresholdPct - forcible checkpoint before end-of-window
* @param negativeTest is this test supposed to fail
* @param numFailures number of failures expected (across all error types); in effect controls number of rollbacks
* @param bootstrapCheckpointsPerWindow k bootstrap checkpoint events are written for every one end-of-window event
* @param timeTakenForDataEventInMs time taken for processing data events
* @param timeTakenForControlEventInMs time taken for processing control events
* @param wrapAround use a smaller producer buffer so that events will wrap around
*/
protected void runDispatcherRollback(int numEvents, int maxWindowSize, int numFailDataEvent, int numFailCheckpointEvent, int numFailEndWindow, double thresholdPct, boolean negativeTest, int numFailures, int bootstrapCheckpointsPerWindow, long timeTakenForDataEventInMs, long timeTakenForControlEventInMs, boolean wrapAround) throws Exception {
LOG.info("Running dispatcher rollback with: " + "numEvents=" + numEvents + " maxWindowSize=" + maxWindowSize + " numFailDataEvent=" + numFailDataEvent + " numFailCheckpoint=" + numFailCheckpointEvent + " numFailEndWindow=" + numFailEndWindow + " thresholdPct=" + thresholdPct + " negativeTest=" + negativeTest + " numFailures=" + numFailures + " bootstrapCheckpointsPerWindow=" + bootstrapCheckpointsPerWindow + " timeTakenForDataEventsInMs=" + timeTakenForDataEventInMs + " timeTakenForControlEventsInMs=" + timeTakenForControlEventInMs + " wrapAround=" + wrapAround);
/* Experiment setup */
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
// set up consumer to fail on data callback at the nth event
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
// single-threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, unifiedStats), null, unifiedStats, null, null);
/* Generate events */
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
double thresholdChkptPct = thresholdPct;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(10);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
// make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = wrapAround ? totalSize : totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/* Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, bootstrapCheckpointsPerWindow == 0);
/* Launch writer */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, bootstrapCheckpointsPerWindow, null);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
// be generous; use worst case for num control events
long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
tEmitter.join(waitTimeMs);
// wait for dispatcher to finish reading the events
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
System.out.println("tConsumer: " + tConsumer);
int windowBeforeDataFail = (numFailDataEvent / maxWindowSize);
int expectedDataFaults = numFailDataEvent == 0 ? 0 : numFailures;
int expectedCheckPointFaults = (numFailCheckpointEvent == 0 || (expectedDataFaults != 0 && numFailCheckpointEvent == windowBeforeDataFail)) ? 0 : numFailures;
// check if all windows were logged by dispatcher; in online case;
if (bootstrapCheckpointsPerWindow == 0) {
Assert.assertTrue(dispatcher.getNumCheckPoints() >= (numCheckpoints - expectedCheckPointFaults));
}
// Consumer prespective
// 1 or 0 faults injected in data callbacks; success (store) differs callback by 1
Assert.assertEquals("Mismatch between callbacks and stored data on consumer.", expectedDataFaults, tConsumer.getDataCallbackCount() - tConsumer.getStoredDataCount());
Assert.assertTrue(tConsumer.getStoredDataCount() >= tConsumer.getNumUniqStoredEvents());
Assert.assertEquals("Consumer failed to store expected number of checkpoints.", dispatcher.getNumCheckPoints(), tConsumer.getStoredCheckpointCount());
// it's converted to ConsumerCallbackResult.SKIP_CHECKPOINT and therefore not seen by client metrics.
if (expectedCheckPointFaults == 0 || expectedDataFaults > 0 || negativeTest) {
Assert.assertTrue("Unexpected error count in consumer metrics (" + unifiedStats.getNumConsumerErrors() + "); should be greater than or equal to numFailures (" + numFailures + ").", unifiedStats.getNumConsumerErrors() >= numFailures);
} else {
Assert.assertEquals("Unexpected error count in consumer metrics; checkpoint errors shouldn't count. ", // unless negativeTest ...
0, unifiedStats.getNumConsumerErrors());
}
// rollback behaviour; were all events re-sent?
if (!negativeTest) {
Assert.assertTrue(tConsumer.getNumUniqStoredEvents() == numEvents);
} else {
Assert.assertTrue(tConsumer.getNumUniqStoredEvents() < numEvents);
}
dispatcher.shutdown();
verifyNoLocks(null, dataEventsBuffer);
}
use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.
the class TestGenericDispatcher method runAbsentSchemaTest.
void runAbsentSchemaTest(boolean setSchemaCheck) throws Exception {
/* Experiment setup */
int numEvents = 100;
int maxWindowSize = 20;
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
//setup consumer to fail on data callback at the nth event
DataDecodingConsumer tConsumer = new DataDecodingConsumer();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
//Single threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
//the schemaIds generated here are random. They will not be the same as those computed in the dispatcher.
//The result is either the processing will fail early (desired behaviour) or during event decoding in the onDataEvent()
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.getDispatcherRetries().setMaxRetryNum(1);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
//make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, false);
//DDSDBUS-3421; set schema check to true
dispatcher.setSchemaIdCheck(setSchemaCheck);
/* Launch writer */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
//be generous ; use worst case for num control events
long waitTimeMs = (numEvents * 1 + numEvents * 1) * 4;
tEmitter.join(waitTimeMs);
//wait for dispatcher to finish reading the events;
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
//asserts
if (!setSchemaCheck) {
//decoding fails many errors show up;
Assert.assertTrue(tConsumer.getNumDataEvents() > 0);
Assert.assertTrue(tConsumer.getNumErrors() > 0);
} else {
//never gets to decoding; but error shows up (exactly one - dispatcher retries set to 1);
Assert.assertEquals(0, tConsumer.getNumDataEvents());
Assert.assertEquals(1, tConsumer.getNumErrors());
}
}
use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.
the class TestGenericDispatcher method testControlEventsRemoval.
@Test(groups = { "small", "functional" })
public void testControlEventsRemoval() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testControlEventsRemoval");
log.info("start");
//DDSDBUS-559
/* Consumer creation */
int timeTakenForEventInMs = 10;
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForEventInMs);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Source configuration */
double thresholdChkptPct = 10.0;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
int numEvents = 100;
int payloadSize = 20;
int maxWindowSize = 1;
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, payloadSize + 62, payloadSize, srcTestEvents) > 0);
long lastWindowScn = srcTestEvents.get(srcTestEvents.size() - 1).sequence();
int size = 0;
for (DbusEvent e : srcTestEvents) {
if (e.size() > size)
size = e.size();
}
//make buffer large enough to hold data
int numWindows = (numEvents / maxWindowSize) + 1;
int producerBufferSize = (numEvents + numWindows) * size + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
final TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("freqCkpt", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, true);
/* Launch writer */
/* write events all of which are empty windows */
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, null, 1.0, true, 0);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
tEmitter.join();
long freeSpaceBefore = dataEventsBuffer.getBufferFreeSpace();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
tDispatcher.join(5000);
LOG.warn("Free Space After=" + dataEventsBuffer.getBufferFreeSpace() + " tConsumer=" + tConsumer + " expected last window=" + lastWindowScn + " last Window = " + dataEventsBuffer.lastWrittenScn());
Assert.assertTrue(dataEventsBuffer.lastWrittenScn() == lastWindowScn);
Assert.assertTrue(freeSpaceBefore < dataEventsBuffer.getBufferFreeSpace());
dispatcher.shutdown();
verifyNoLocks(null, dataEventsBuffer);
log.info("end\n");
}
Aggregations