use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.
the class DatabusHttpClientImpl method initializeRelayConnections.
private synchronized void initializeRelayConnections() {
for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
// nothing to do
if (null == relayConsumers || 0 == relayConsumers.size())
continue;
try {
DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
if (null == connConfig) {
connConfig = getClientStaticConfig().getConnectionDefaults();
}
// make sure we have the right policy.
if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
}
CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
cpPersistenceProvder.removeCheckpoint(sourcesStrList);
}
ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
if (server0.supportsSources(bstConsumerReg.getSources())) {
bstConsumersRegs.add(bstConsumerReg);
}
}
}
DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
eventBuffer.setDropOldEvents(true);
eventBuffer.start(0);
DbusEventBuffer bootstrapBuffer = null;
// create bootstrap only if it is enabled
if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
bootstrapBuffer.setDropOldEvents(false);
bootstrapBuffer.start(0);
}
LOG.info("The sourcesList is " + sourcesStrList);
LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
Set<ServerInfo> relays = _relayGroups.get(subsList);
Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
String statsCollectorName = generateSubsStatsName(sourcesStrList);
int ownerId = getContainerStaticConfig().getId();
_bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
_inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
_outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
_consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
_bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
_unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, // _relayGroupBootstrapConsumers.get(sourcesList),
bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
newConn.start();
_relayConnections.add(newConn);
} catch (Exception e) {
LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
}
}
if (0 == _relayConnections.size()) {
LOG.warn("No connections specified");
}
}
use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.
the class TestMultiConsumerCallback method testPerf.
@Test
public void testPerf() throws Exception {
LOG.info("\n\nstarting testPerf()");
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
eventsBuf.endEvents(100L);
DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
assert iter.hasNext() : "unable to read event";
iter.next();
DbusEvent event1 = iter.next();
DatabusStreamConsumer logConsumer = new LoggingConsumer();
SelectingDatabusCombinedConsumer sdccLogConsumer = new SelectingDatabusCombinedConsumer(logConsumer);
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccLogConsumer, sources, null);
ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newCachedThreadPool();
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, executor, 60000, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, null, null);
callback.setSourceMap(sourcesMap);
callback.onStartConsumption();
callback.onStartDataEventSequence(new SingleSourceSCN(1, 1));
for (int i = 0; i < 10000; ++i) {
callback.onDataEvent(event1, null);
}
callback.onEndDataEventSequence(new SingleSourceSCN(1, 1));
callback.onStopConsumption();
System.out.println("max threads=" + executor.getLargestPoolSize() + " task count=" + executor.getTaskCount());
System.out.println("dataEventsReceived=" + consumerStatsCollector.getNumDataEventsReceived() + " sysEventsReceived=" + consumerStatsCollector.getNumSysEventsReceived() + " dataEventsProcessed=" + consumerStatsCollector.getNumDataEventsProcessed() + " latencyEventsProcessed=" + consumerStatsCollector.getLatencyEventsProcessed());
long dataEvents = consumerStatsCollector.getNumDataEventsReceived();
assert (consumerStatsCollector.getNumDataEventsProcessed() == dataEvents);
}
use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.
the class TestGenericDispatcher method testShutdownBeforeRollback.
@Test(groups = { "small", "functional" })
public /**
* 1. Dispatcher is dispatching 2 window of events.
* 2. First window consumption is successfully done.
* 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
* causing the dispatcher to wait.
* 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
* cause rollback in normal scenario.
* 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
* calls MUST be made.
*/
void testShutdownBeforeRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
log.setLevel(Level.INFO);
// log.getRoot().setLevel(Level.DEBUG);
LOG.info("start");
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 8;
final int numEventsPerWindow = 4;
final int payloadSize = 200;
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
for (DbusEvent e : srcTestEvents) {
win1Size += e.size();
}
// serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
// Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
/**
* Consumer with ability to wait for latch during onStartDataEventSequence()
*/
class TimeoutDESConsumer extends TimeoutTestConsumer {
private final CountDownLatch latch = new CountDownLatch(1);
private int _countStartWindow = 0;
private final int _failedRequestNumber;
public TimeoutDESConsumer(int failedRequestNumber) {
super(1, 1, 0, 0, 0, 0);
_failedRequestNumber = failedRequestNumber;
}
public CountDownLatch getLatch() {
return latch;
}
@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
_countStartWindow++;
if (_countStartWindow == _failedRequestNumber) {
// Wait for the latch to open
try {
latch.await();
} catch (InterruptedException e) {
}
return ConsumerCallbackResult.ERROR_FATAL;
}
return super.onStartDataEventSequence(startScn);
}
@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
return ConsumerCallbackResult.SUCCESS;
}
public int getNumBeginWindowCalls() {
return _countStartWindow;
}
}
// Create dispatcher
// fail on second window
final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
final Thread dispatcherThread = new Thread(dispatcher);
log.info("starting dispatcher thread");
dispatcherThread.start();
// Generate RegisterRespone for schema
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
// Enqueue Necessary messages before starting dispatch
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
// comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
/*
* Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
* With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
* reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
*/
readerStream.configureBlocking(false);
// Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
// Event readers - Clients in the real world
// Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send both windows");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
// EOP events, presumably?
Assert.assertEquals(// EOP events, presumably?
"num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 2 == mockConsumer.getNumBeginWindowCalls();
}
}, "second window processing started", 5000, log);
dispatcher.shutdown();
// remove the barrier
mockConsumer.getLatch().countDown();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return !dispatcherThread.isAlive();
}
}, "Ensure Dispatcher thread is shutdown", 5000, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 0 == mockConsumer.getNumRollbacks();
}
}, "Ensure No Rollback is called", 10, log);
} finally {
reader.stop();
}
log.info("end\n");
}
use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.
the class DatabusV2RegistrationImpl method initializeStatsCollectors.
/**
* Initialize Statistics Collectors
*/
protected void initializeStatsCollectors(String regId, int ownerId, MBeanServer mbeanServer) {
_inboundEventsStatsCollector = new DbusEventsStatisticsCollector(ownerId, regId + STREAM_EVENT_STATS_SUFFIX_NAME, true, false, mbeanServer);
_bootstrapEventsStatsCollector = new DbusEventsStatisticsCollector(ownerId, regId + BOOTSTRAP_EVENT_STATS_SUFFIX_NAME, true, false, mbeanServer);
_relayConsumerStats = new ConsumerCallbackStats(ownerId, regId + RELAY_CONSUMER_STATS_SUFFIX_NAME, regId, true, false, new ConsumerCallbackStatsEvent());
_bootstrapConsumerStats = new ConsumerCallbackStats(ownerId, regId + BOOTSTRAP_CONSUMER_STATS_SUFFIX_NAME, regId, true, false, new ConsumerCallbackStatsEvent());
_unifiedClientStats = new UnifiedClientStats(ownerId, regId + UNIFIED_CLIENT_STATS_SUFFIX_NAME, regId, true, false, _client.getClientStaticConfig().getPullerThreadDeadnessThresholdMs(), new UnifiedClientStatsEvent());
}
Aggregations