use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testOneWindowHappyPath.
@Test(groups = { "small", "functional" })
public void testOneWindowHappyPath() {
final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowHappyPath");
// log.setLevel(Level.DEBUG);
log.info("start");
int source1EventsNum = 2;
int source2EventsNum = 2;
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
// svsConsumer.getLog().setLevel(Level.DEBUG);
DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
dispatcher.setSchemaIdCheck(false);
Thread dispatcherThread = new Thread(dispatcher, "testOneWindowHappyPath-dispatcher");
// dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
eventsBuf.endEvents(100L, null);
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
}
dispatcher.shutdown();
Checkpoint cp = dispatcher.getDispatcherState().getLastSuccessfulCheckpoint();
Assert.assertEquals(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET, cp.getWindowOffset());
Assert.assertEquals(cp.getWindowScn(), cp.getPrevScn());
for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
}
assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
verifyNoLocks(log, eventsBuf);
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testPartialWindowRollback.
@Test(groups = { "small", "functional" })
public /**
* Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
* with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
* is correctly triggered.
*
* The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
* callback.
*
* 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
* 2) Read partial second window: e2_1 e2_2
* 3) The above should fail -- verify that rollback is called
* 4) Read the rest
*/
void testPartialWindowRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
// log.setLevel(Level.INFO);
log.info("start");
final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
// Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 9;
// 1-based number of the event callback to fail
final int numOfFailureEvent = 5;
final int numEventsPerWindow = 3;
final int payloadSize = 200;
final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
int win2Size = 0;
int eventN = 0;
for (DbusEvent e : srcTestEvents) {
eventN++;
if (eventN <= numEventsPerWindow) {
win1Size += e.size();
} else if (eventN <= numOfFailureEvent) {
win2Size += e.size();
}
}
// serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
// Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
// Create dispatcher
final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
dispatcher.setSchemaIdCheck(false);
Thread dispatcherThread = new Thread(dispatcher);
dispatcherThread.setDaemon(true);
log.info("starting dispatcher thread");
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
// stream the events from the source buffer without the EOW
// comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
readerStream.configureBlocking(false);
// Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
// Event readers - Clients in the real world
// Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send first window -- that one should be OK");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == callbackStats.getNumSysEventsProcessed();
}
}, "first window processed", 5000, log);
log.info("send the second partial window -- that one should cause an error");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
log.info("wait for dispatcher to finish");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("events received: " + callbackStats.getNumDataEventsReceived());
return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
}
}, "all events until the error processed", 5000, log);
log.info("all data events have been received but no EOW");
Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
// at least one failing event therefore < numOfFailureEvent events can be processed
Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
// onDataEvent callbacks for e2_1 and e2_2 get cancelled
Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
// only one EOW
Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
log.info("Send the remainder of the window");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
// remaining events + EOWs
Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
log.info("wait for the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == mockConsumer.getNumRollbacks();
}
}, "rollback seen", 5000, log);
log.info("wait for dispatcher to finish after the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
return numWindows == callbackStats.getNumSysEventsProcessed();
}
}, "all events processed", 5000, log);
} finally {
reader.stop();
dispatcher.shutdown();
log.info("all events processed");
verifyNoLocks(null, srcEventsBuf);
verifyNoLocks(null, destEventsBuf);
}
Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testRollbackFailure.
@Test(groups = { "small", "functional" })
public /**
* Test a relatively big window which forces a checkpoint which fails. Then trigger
* a rollback. The dispatcher should shutdown.
*/
void testRollbackFailure() throws InvalidConfigException {
final Logger log = Logger.getLogger("TestGenericDispatcher.testRollbackFailure");
// log.setLevel(Level.DEBUG);
log.info("start");
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, 100, (short) 1, null, null);
eventsBuf.endEvents(100L, null);
RollbackFailingConsumer mockConsumer = new RollbackFailingConsumer(new StateVerifyingStreamConsumer(null), LOG, 2);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(mockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
DatabusSourcesConnection.Config connCfgBuilder = new DatabusSourcesConnection.Config();
connCfgBuilder.setConsumerParallelism(1);
connCfgBuilder.setEventBuffer(_generic100KBufferConfig);
connCfgBuilder.setFreeBufferThreshold(10000);
connCfgBuilder.setConsumerTimeBudgetMs(1000);
connCfgBuilder.setCheckpointThresholdPct(5.0);
connCfgBuilder.getDispatcherRetries().setMaxRetryNum(10);
DatabusSourcesConnection.StaticConfig connStaticCfg = connCfgBuilder.build();
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", connStaticCfg, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
Thread dispatcherThread = new Thread(dispatcher);
// dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
try {
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
LOG.info(dispatcher.getStatus().getStatus().toString());
LOG.info(String.valueOf(dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN));
return dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN;
}
}, "dispatcher shtudown", 50000, LOG);
} finally {
dispatcher.shutdown();
}
verifyNoLocks(null, eventsBuf);
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testMetadataSchema.
@Test
public void testMetadataSchema() {
final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
// log.setLevel(Level.DEBUG);
log.info("start");
int source1EventsNum = 2;
int source2EventsNum = 2;
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
// svsConsumer.getLog().setLevel(Level.DEBUG);
DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
// dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
// add meta data schema
byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
eventsBuf.endEvents(100L, null);
// check standard execution of callbacks
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
}
dispatcher.shutdown();
for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
}
assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
// check metadata schemas
EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
VersionedSchema metadataSchema = myCons.getMetadataSchema();
Assert.assertTrue(null != metadataSchema);
log.info("Metadata VersionedSchema = " + metadataSchema);
Assert.assertEquals(metadataSchema.getVersion(), 2);
Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
verifyNoLocks(log, eventsBuf);
log.info("end\n");
}
use of com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory in project databus by linkedin.
the class TestGenericDispatcher method testMultiWindowsHappyPath.
@Test(groups = { "small", "functional" })
public void testMultiWindowsHappyPath() {
final Logger log = Logger.getLogger("TestGenericDispatcher.testMultiWindowsHappyPath");
// log.setLevel(Level.DEBUG);
log.info("start");
int source1EventsNum = 3;
int source2EventsNum = 5;
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
int windowsNum = 3;
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
int curEventNum = 1;
for (int w = 0; w < windowsNum; ++w) {
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, curEventNum, source1EventsNum, (short) 1, keyCounts, srcidCounts);
curEventNum += source1EventsNum;
initBufferWithEvents(eventsBuf, curEventNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
curEventNum += source2EventsNum;
eventsBuf.endEvents(100L * (w + 1), null);
}
DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
Thread dispatcherThread = new Thread(dispatcher);
// dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
}
dispatcher.shutdown();
for (long i = 1; i < curEventNum; ++i) {
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
}
assertEquals("correct amount of callbacks for srcid 1", windowsNum * source1EventsNum, srcidCounts.get((short) 1).intValue());
assertEquals("correct amount of callbacks for srcid 2", windowsNum * source2EventsNum, srcidCounts.get((short) 2).intValue());
verifyNoLocks(null, eventsBuf);
log.info("end\n");
}
Aggregations