Examples with ConsumerCallbackStats com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats used on opensource projects

Example 1 with ConsumerCallbackStats

use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    // this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    // as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    // client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    // get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            // unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}

Example 2 with ConsumerCallbackStats

use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.

the class TestGenericDispatcher method testPartialWindowRollback.

@Test(groups = { "small", "functional" })
public /**
 * Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
 * with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
 * is correctly triggered.
 *
 * The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
 * callback.
 *
 * 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
 * 2) Read partial second window: e2_1 e2_2
 * 3) The above should fail -- verify that rollback is called
 * 4) Read the rest
 */
void testPartialWindowRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
    // log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    // Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 9;
    // 1-based number of the event callback to fail
    final int numOfFailureEvent = 5;
    final int numEventsPerWindow = 3;
    final int payloadSize = 200;
    final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    int win2Size = 0;
    int eventN = 0;
    for (DbusEvent e : srcTestEvents) {
        eventN++;
        if (eventN <= numEventsPerWindow) {
            win1Size += e.size();
        } else if (eventN <= numOfFailureEvent) {
            win2Size += e.size();
        }
    }
    // serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    // Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    // Create dispatcher
    final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    dispatcher.setSchemaIdCheck(false);
    Thread dispatcherThread = new Thread(dispatcher);
    dispatcherThread.setDaemon(true);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    // stream the events from the source buffer without the EOW
    // comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    readerStream.configureBlocking(false);
    // Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    // Event readers - Clients in the real world
    // Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send first window -- that one should be OK");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == callbackStats.getNumSysEventsProcessed();
            }
        }, "first window processed", 5000, log);
        log.info("send the second partial window -- that one should cause an error");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
        Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
        log.info("wait for dispatcher to finish");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("events received: " + callbackStats.getNumDataEventsReceived());
                return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
            }
        }, "all events until the error processed", 5000, log);
        log.info("all data events have been received but no EOW");
        Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
        Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
        // at least one failing event therefore < numOfFailureEvent events can be processed
        Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
        // onDataEvent callbacks for e2_1 and e2_2 get cancelled
        Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
        // only one EOW
        Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
        log.info("Send the remainder of the window");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
        // remaining events + EOWs
        Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
        log.info("wait for the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 1 == mockConsumer.getNumRollbacks();
            }
        }, "rollback seen", 5000, log);
        log.info("wait for dispatcher to finish after the rollback");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
                return numWindows == callbackStats.getNumSysEventsProcessed();
            }
        }, "all events processed", 5000, log);
    } finally {
        reader.stop();
        dispatcher.shutdown();
        log.info("all events processed");
        verifyNoLocks(null, srcEventsBuf);
        verifyNoLocks(null, destEventsBuf);
    }
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}

Example 3 with ConsumerCallbackStats

use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.

the class DatabusV2ClusterRegistrationImpl method initializeStatsCollectors.

protected synchronized void initializeStatsCollectors() {
    // some safety against null pointers coming from unit tests
    MBeanServer mbeanServer = null;
    int ownerId = -1;
    long pullerThreadDeadnessThresholdMs = UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS;
    if (null != _client) {
        mbeanServer = _client.getMbeanServer();
        ownerId = _client.getContainerStaticConfig().getId();
        pullerThreadDeadnessThresholdMs = _client.getClientStaticConfig().getPullerThreadDeadnessThresholdMs();
    }
    String regId = null != _id ? _id.getId() : "unknownReg";
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(ownerId, regId + ".callback.relay", regId, true, false, new ConsumerCallbackStatsEvent());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(ownerId, regId + ".callback.bootstrap", regId, true, false, new ConsumerCallbackStatsEvent());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(ownerId, regId + ".callback.unified", regId, true, false, pullerThreadDeadnessThresholdMs, new UnifiedClientStatsEvent());
    _relayCallbackStatsMerger = new StatsCollectors<ConsumerCallbackStats>(relayConsumerStats);
    _bootstrapCallbackStatsMerger = new StatsCollectors<ConsumerCallbackStats>(bootstrapConsumerStats);
    _unifiedClientStatsMerger = new StatsCollectors<UnifiedClientStats>(unifiedClientStats);
    _relayEventStatsMerger = new StatsCollectors<DbusEventsStatisticsCollector>(new AggregatedDbusEventsStatisticsCollector(ownerId, regId + ".inbound", true, false, mbeanServer));
    _bootstrapEventStatsMerger = new StatsCollectors<DbusEventsStatisticsCollector>(new AggregatedDbusEventsStatisticsCollector(ownerId, regId + ".inbound.bs", true, false, mbeanServer));
    if (null != _client) {
        _client.getBootstrapEventsStats().addStatsCollector(regId, _bootstrapEventStatsMerger.getStatsCollector());
        _client.getInBoundStatsCollectors().addStatsCollector(regId, _relayEventStatsMerger.getStatsCollector());
        _client.getRelayConsumerStatsCollectors().addStatsCollector(regId, _relayCallbackStatsMerger.getStatsCollector());
        _client.getBootstrapConsumerStatsCollectors().addStatsCollector(regId, _bootstrapCallbackStatsMerger.getStatsCollector());
        _client.getUnifiedClientStatsCollectors().addStatsCollector(regId, _unifiedClientStatsMerger.getStatsCollector());
        _client.getGlobalStatsMerger().registerStatsCollector(_relayEventStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_bootstrapEventStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_relayCallbackStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_bootstrapCallbackStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_unifiedClientStatsMerger);
    }
}

Example 4 with ConsumerCallbackStats

use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.

the class TestMultiConsumerCallback method test1ConsumerTimeout.

@Test
public void test1ConsumerTimeout() {
    LOG.info("\n\nstarting test1ConsumerTimeout()");
    // create dummy events
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    // skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event3 = iter.next().createCopy();
    // make up some sources
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    // create the consumer mock up
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 150));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 110));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 40));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 50));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 210));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 50));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    // Create and fire up callbacks
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 100, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, null, null);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startConsumptionRes), "startConsumption() failed");
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startWindowRes), "startDataEventSequence() failed");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    EasyMock.expect(mockConsumer1.onStartSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 150)).times(0, 1);
    EasyMock.expect(mockConsumer1.onDataEvent(event3, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(3) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 60));
    EasyMock.replay(mockConsumer1);
    startSourceRes = callback.onStartSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startSourceRes), "startSources(source2) fails");
    event1Res = callback.onDataEvent(event3, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(3) succeeded");
    endSourceRes = callback.onEndSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(endSourceRes), "onEndSource succeeds");
    long eventsErrProcessed = consumerStatsCollector.getNumErrorsProcessed();
    long totalEvents = consumerStatsCollector.getNumEventsReceived();
    long totalEventsProcessed = consumerStatsCollector.getNumEventsProcessed();
    System.out.println("eventsReceived = " + consumerStatsCollector.getNumEventsReceived() + " eventsProcessed=" + consumerStatsCollector.getNumEventsProcessed());
    System.out.println("eventsErrProcessed =" + consumerStatsCollector.getNumErrorsProcessed() + " eventsErrReceived=" + consumerStatsCollector.getNumErrorsReceived() + " totalEvents=" + consumerStatsCollector.getNumEventsReceived() + " totalEventsProcessed=" + totalEventsProcessed);
    // FIXME
    Assert.assertTrue(totalEvents >= totalEventsProcessed + eventsErrProcessed);
    Assert.assertTrue(eventsErrProcessed > 0);
    Assert.assertTrue(totalEventsProcessed < totalEvents);
// NOTE: We don't verify because all the canceled callbacks are not detected by EasyMock
// EasyMock.verify(mockConsumer1);
}

Example 5 with ConsumerCallbackStats

use of com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats in project databus by linkedin.

the class TestMultiConsumerCallback method test2ConsumerTimeout.

@Test
public void test2ConsumerTimeout() {
    Logger log = Logger.getLogger("TestMultiConsumerCallback.test2ConsumerTimeout");
    // Logger.getRootLogger().setLevel(Level.INFO);
    log.info("\n\nstarting test2ConsumerTimeout()");
    log.info("create dummy events");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    // skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    log.info("make up some sources");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    log.info("create the consumer mock up");
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 1));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 1));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    log.info("Create and fire up callbacks");
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    TimingOutMultiConsumerCallback callback = new TimingOutMultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 300, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, 3);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startConsumptionRes), "startConsumption() succeeded: " + startConsumptionRes);
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startWindowRes), "startDataEventSequence() succeeded");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    log.info("test2ConsumerTimeout: end");
}