Search in sources :

Example 6 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class RelayPullThread method doReadDataEvents.

protected void doReadDataEvents(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    boolean enqueueMessage = true;
    try {
        ChunkedBodyReadableByteChannel readChannel = curState.getReadChannel();
        Checkpoint cp = curState.getCheckpoint();
        curState.setRelayFellOff(false);
        String remoteErrorName = RemoteExceptionHandler.getExceptionName(readChannel);
        Throwable knownRemoteError = _remoteExceptionHandler.getException(readChannel);
        if (null != knownRemoteError && knownRemoteError instanceof ScnNotFoundException) {
            if (toTearConnAfterHandlingResponse()) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                curState.setRelayFellOff(true);
                if (_retriesOnFallOff.getRemainingRetriesNum() > 0) {
                    _log.error("Got SCNNotFoundException. Retry (" + _retriesOnFallOff.getRetriesNum() + ") out of " + _retriesOnFallOff.getConfig().getMaxRetryNum());
                    curState.switchToPickServer();
                } else {
                    enqueueMessage = onRelayFellOff(curState, cp, knownRemoteError);
                }
            }
        } else if (null != remoteErrorName) {
            if (toTearConnAfterHandlingResponse()) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                //remote processing error
                _log.error("read events error: " + RemoteExceptionHandler.getExceptionMessage(readChannel));
                curState.switchToStreamResponseError();
            }
        } else {
            /*DispatcherState dispatchState = curState.getDispatcherState();
          dispatchState.switchToDispatchEvents();
          _dispatcherThread.addNewStateBlocking(dispatchState);*/
            if (debugEnabled)
                _log.debug("Sending events to buffer");
            DbusEventsStatisticsCollector connCollector = _sourcesConn.getInboundEventsStatsCollector();
            if (curState.isSCNRegress()) {
                _log.info("SCN Regress requested !! Sending a SCN Regress Message to dispatcher. Curr Ckpt :" + curState.getCheckpoint());
                DbusEvent regressEvent = getEventFactory().createSCNRegressEvent(new SCNRegressMessage(curState.getCheckpoint()));
                writeEventToRelayDispatcher(curState, regressEvent, "SCN Regress Event from ckpt :" + curState.getCheckpoint());
                curState.setSCNRegress(false);
            }
            UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
            if (unifiedClientStats != null) {
                // failsafe:  we're definitely not bootstrapping here
                unifiedClientStats.setBootstrappingState(false);
                sendHeartbeat(unifiedClientStats);
            }
            int eventsNum = curState.getDataEventsBuffer().readEvents(readChannel, curState.getListeners(), connCollector);
            boolean resetConnection = false;
            if (eventsNum > 0) {
                _timeSinceEventsSec = System.currentTimeMillis();
                cp.checkPoint();
            } else {
                // check how long it has been since we got some events
                if (_remoteExceptionHandler.getPendingEventSize(readChannel) > curState.getDataEventsBuffer().getMaxReadBufferCapacity()) {
                    // The relay had a pending event that we can never accommodate. This is fatal error.
                    String err = "ReadBuffer max capacity(" + curState.getDataEventsBuffer().getMaxReadBufferCapacity() + ") is less than event size(" + _remoteExceptionHandler.getPendingEventSize(readChannel) + "). Increase databus.client.connectionDefaults.eventBuffer.maxEventSize and restart.";
                    _log.fatal(err);
                    enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new PendingEventTooLargeException(err)));
                    return;
                } else {
                    if (_noEventsConnectionResetTimeSec > 0) {
                        // unless the feature is disabled (<=0)
                        resetConnection = (System.currentTimeMillis() - _timeSinceEventsSec) / 1000 > _noEventsConnectionResetTimeSec;
                        if (resetConnection) {
                            _timeSinceEventsSec = System.currentTimeMillis();
                            _log.warn("about to reset connection to relay " + curState.getServerInetAddress() + ", because there were no events for " + _noEventsConnectionResetTimeSec + "secs");
                        }
                    }
                }
            }
            if (debugEnabled)
                _log.debug("Events read: " + eventsNum);
            // if it has been too long since we got non-empty responses - the relay may be stuck, try to reconnect
            if (toTearConnAfterHandlingResponse() || resetConnection) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                curState.switchToStreamResponseDone();
                resetServerRetries();
            }
        }
        if (enqueueMessage)
            enqueueMessage(curState);
    } catch (InterruptedException ie) {
        _log.warn("interrupted", ie);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    } catch (InvalidEventException e) {
        _log.error("error reading events from server:" + e, e);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    } catch (RuntimeException e) {
        _log.error("runtime error reading events from server: " + e, e);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    }
}
Also used : UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) PendingEventTooLargeException(com.linkedin.databus.core.PendingEventTooLargeException) DbusEvent(com.linkedin.databus.core.DbusEvent) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) Checkpoint(com.linkedin.databus.core.Checkpoint) ScnNotFoundException(com.linkedin.databus.core.ScnNotFoundException) SCNRegressMessage(com.linkedin.databus.core.SCNRegressMessage) InvalidEventException(com.linkedin.databus.core.InvalidEventException)

Example 7 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class RelayPullThread method doBootstrapComplete.

private void doBootstrapComplete(BootstrapResultMessage bootstrapResultMessage) {
    Checkpoint cp = bootstrapResultMessage.getBootstrapCheckpoint();
    // we must have persisted some checkpoint at the end of the bootstrap phase
    if (null == cp) {
        bootstrapResultMessage.switchToBootstrapFailed(new RuntimeException("No persistent checkpoint found at the end of bootstrap!"));
        doBootstrapFailed(bootstrapResultMessage);
    } else {
        _currentState.setRelayFellOff(false);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        _log.info("Bootstrap completed: " + "Consumption Mode=" + consumptionMode + " startScn=" + cp.getBootstrapStartScn() + " targetScn=" + cp.getBootstrapTargetScn() + " sinceScn=" + cp.getBootstrapSinceScn() + " windowScn=" + cp.getWindowScn());
        UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
        if (unifiedClientStats != null) {
            // should always be false, but just in case of weird edge cases:
            boolean isBootstrapping = (consumptionMode == DbusClientMode.BOOTSTRAP_SNAPSHOT || consumptionMode == DbusClientMode.BOOTSTRAP_CATCHUP);
            unifiedClientStats.setBootstrappingState(isBootstrapping);
        }
        // clear Bootstrap scns for future bootstraps
        cp.resetBootstrap();
        if (toTearConnAfterHandlingResponse()) {
            tearConnectionAndEnqueuePickServer();
        } else {
            _currentState.switchToRequestStream(cp);
            enqueueMessage(_currentState);
        }
    }
}
Also used : Checkpoint(com.linkedin.databus.core.Checkpoint) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusClientMode(com.linkedin.databus.core.DbusClientMode)

Example 8 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class TestMultiConsumerCallback method test1ConsumerTimeout.

@Test
public void test1ConsumerTimeout() {
    LOG.info("\n\nstarting test1ConsumerTimeout()");
    //create dummy events
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    //skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event3 = iter.next().createCopy();
    //make up some sources
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    //create the consumer mock up
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 150));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 110));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 40));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 50));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 210));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 50));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    //Create and fire up callbacks
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 100, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, null, null);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startConsumptionRes), "startConsumption() failed");
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startWindowRes), "startDataEventSequence() failed");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    EasyMock.expect(mockConsumer1.onStartSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 150)).times(0, 1);
    EasyMock.expect(mockConsumer1.onDataEvent(event3, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(3) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 60));
    EasyMock.replay(mockConsumer1);
    startSourceRes = callback.onStartSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startSourceRes), "startSources(source2) fails");
    event1Res = callback.onDataEvent(event3, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(3) succeeded");
    endSourceRes = callback.onEndSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(endSourceRes), "onEndSource succeeds");
    long eventsErrProcessed = consumerStatsCollector.getNumErrorsProcessed();
    long totalEvents = consumerStatsCollector.getNumEventsReceived();
    long totalEventsProcessed = consumerStatsCollector.getNumEventsProcessed();
    System.out.println("eventsReceived = " + consumerStatsCollector.getNumEventsReceived() + " eventsProcessed=" + consumerStatsCollector.getNumEventsProcessed());
    System.out.println("eventsErrProcessed =" + consumerStatsCollector.getNumErrorsProcessed() + " eventsErrReceived=" + consumerStatsCollector.getNumErrorsReceived() + " totalEvents=" + consumerStatsCollector.getNumEventsReceived() + " totalEventsProcessed=" + totalEventsProcessed);
    //FIXME
    Assert.assertTrue(totalEvents >= totalEventsProcessed + eventsErrProcessed);
    Assert.assertTrue(eventsErrProcessed > 0);
    Assert.assertTrue(totalEventsProcessed < totalEvents);
//NOTE: We don't verify because all the canceled callbacks are not detected by EasyMock
//EasyMock.verify(mockConsumer1);
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) IdNamePair(com.linkedin.databus.core.util.IdNamePair) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) Hashtable(java.util.Hashtable) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 9 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class TestMultiConsumerCallback method test2ConsumerTimeout.

@Test
public void test2ConsumerTimeout() {
    Logger log = Logger.getLogger("TestMultiConsumerCallback.test2ConsumerTimeout");
    //Logger.getRootLogger().setLevel(Level.INFO);
    log.info("\n\nstarting test2ConsumerTimeout()");
    log.info("create dummy events");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    //skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    log.info("make up some sources");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    log.info("create the consumer mock up");
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 1));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 1));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    log.info("Create and fire up callbacks");
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    TimingOutMultiConsumerCallback callback = new TimingOutMultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 300, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, 3);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startConsumptionRes), "startConsumption() succeeded: " + startConsumptionRes);
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startWindowRes), "startDataEventSequence() succeeded");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    log.info("test2ConsumerTimeout: end");
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) Hashtable(java.util.Hashtable) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 10 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class DatabusV2ClusterRegistrationImpl method initializeStatsCollectors.

protected synchronized void initializeStatsCollectors() {
    //some safety against null pointers coming from unit tests
    MBeanServer mbeanServer = null;
    int ownerId = -1;
    long pullerThreadDeadnessThresholdMs = UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS;
    if (null != _client) {
        mbeanServer = _client.getMbeanServer();
        ownerId = _client.getContainerStaticConfig().getId();
        pullerThreadDeadnessThresholdMs = _client.getClientStaticConfig().getPullerThreadDeadnessThresholdMs();
    }
    String regId = null != _id ? _id.getId() : "unknownReg";
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(ownerId, regId + ".callback.relay", regId, true, false, new ConsumerCallbackStatsEvent());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(ownerId, regId + ".callback.bootstrap", regId, true, false, new ConsumerCallbackStatsEvent());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(ownerId, regId + ".callback.unified", regId, true, false, pullerThreadDeadnessThresholdMs, new UnifiedClientStatsEvent());
    _relayCallbackStatsMerger = new StatsCollectors<ConsumerCallbackStats>(relayConsumerStats);
    _bootstrapCallbackStatsMerger = new StatsCollectors<ConsumerCallbackStats>(bootstrapConsumerStats);
    _unifiedClientStatsMerger = new StatsCollectors<UnifiedClientStats>(unifiedClientStats);
    _relayEventStatsMerger = new StatsCollectors<DbusEventsStatisticsCollector>(new AggregatedDbusEventsStatisticsCollector(ownerId, regId + ".inbound", true, false, mbeanServer));
    _bootstrapEventStatsMerger = new StatsCollectors<DbusEventsStatisticsCollector>(new AggregatedDbusEventsStatisticsCollector(ownerId, regId + ".inbound.bs", true, false, mbeanServer));
    if (null != _client) {
        _client.getBootstrapEventsStats().addStatsCollector(regId, _bootstrapEventStatsMerger.getStatsCollector());
        _client.getInBoundStatsCollectors().addStatsCollector(regId, _relayEventStatsMerger.getStatsCollector());
        _client.getRelayConsumerStatsCollectors().addStatsCollector(regId, _relayCallbackStatsMerger.getStatsCollector());
        _client.getBootstrapConsumerStatsCollectors().addStatsCollector(regId, _bootstrapCallbackStatsMerger.getStatsCollector());
        _client.getUnifiedClientStatsCollectors().addStatsCollector(regId, _unifiedClientStatsMerger.getStatsCollector());
        _client.getGlobalStatsMerger().registerStatsCollector(_relayEventStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_bootstrapEventStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_relayCallbackStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_bootstrapCallbackStatsMerger);
        _client.getGlobalStatsMerger().registerStatsCollector(_unifiedClientStatsMerger);
    }
}
Also used : UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) ConsumerCallbackStatsEvent(com.linkedin.databus.client.pub.monitoring.events.ConsumerCallbackStatsEvent) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) Checkpoint(com.linkedin.databus.core.Checkpoint) UnifiedClientStatsEvent(com.linkedin.databus.client.pub.monitoring.events.UnifiedClientStatsEvent) MBeanServer(javax.management.MBeanServer)

Aggregations

UnifiedClientStats (com.linkedin.databus.client.pub.mbean.UnifiedClientStats)21 Test (org.testng.annotations.Test)10 ConsumerCallbackStats (com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats)9 Checkpoint (com.linkedin.databus.core.Checkpoint)9 DbusEvent (com.linkedin.databus.core.DbusEvent)9 ArrayList (java.util.ArrayList)7 DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)6 DbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)6 IdNamePair (com.linkedin.databus.core.util.IdNamePair)6 HashMap (java.util.HashMap)6 DatabusV2ConsumerRegistration (com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration)5 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)5 MultiConsumerCallback (com.linkedin.databus.client.consumer.MultiConsumerCallback)3 SelectingDatabusCombinedConsumer (com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer)3 StreamConsumerCallbackFactory (com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory)3 DatabusStreamConsumer (com.linkedin.databus.client.pub.DatabusStreamConsumer)3 ConsumerCallbackResult (com.linkedin.databus.client.pub.ConsumerCallbackResult)2 ServerInfo (com.linkedin.databus.client.pub.ServerInfo)2 ConsumerCallbackStatsEvent (com.linkedin.databus.client.pub.monitoring.events.ConsumerCallbackStatsEvent)2 UnifiedClientStatsEvent (com.linkedin.databus.client.pub.monitoring.events.UnifiedClientStatsEvent)2