Search in sources :

Example 21 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class TestMultiConsumerCallback method test3GroupedStreamConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void test3GroupedStreamConsumersHappyPath() {
    LOG.info("\n\nstarting test3GroupedStreamConsumersHappyPath()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusStreamConsumer mockConsumer2 = EasyMock.createStrictMock("consumer2", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer2, true);
    DatabusStreamConsumer mockConsumer3 = EasyMock.createStrictMock("consumer3", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer3, true);
    DatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    DatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    DatabusCombinedConsumer sdccMockConsumer3 = new SelectingDatabusCombinedConsumer(mockConsumer3);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg1 = new DatabusV2ConsumerRegistration(Arrays.asList(sdccMockConsumer1, sdccMockConsumer2, sdccMockConsumer3), sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg1);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer1, event1, event2, event3, keyCounts);
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer2, event1, event2, event3, keyCounts);
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer3, event1, event2, event3, keyCounts);
    assert3EventFullLifecycle(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer1);
    EasyMock.verify(mockConsumer2);
    EasyMock.verify(mockConsumer3);
    assert (keyCounts.get(1L).get() + keyCounts.get(2L).get() + keyCounts.get(3L).get()) == 3 : "invalid number of calls: " + keyCounts.get(1L).get() + "," + keyCounts.get(2L).get() + "," + keyCounts.get(3L).get();
}
Also used : DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) DbusEvent(com.linkedin.databus.core.DbusEvent) HashMap(java.util.HashMap) Hashtable(java.util.Hashtable) ArrayList(java.util.ArrayList) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 22 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onStartSource.

@Override
public ConsumerCallbackResult onStartSource(String source, Schema sourceSchema) {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> startSourceCallable = _callbackFactory.createStartSourceCallable(curNanos, source, sourceSchema, consumer, true);
            _currentBatch.add(startSourceCallable);
            if (_consumerStats != null)
                _consumerStats.registerEventsReceived(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> startSourceCallable = _callbackFactory.createStartSourceCallable(curNanos, source, sourceSchema, _loggingConsumer, false);
        _currentBatch.add(startSourceCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onStartSource = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, false, true);
}
Also used : ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer)

Example 23 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onError.

@Override
public ConsumerCallbackResult onError(Throwable err) {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> onErrorCallable = _callbackFactory.createOnErrorCallable(curNanos, err, consumer, true);
            _currentBatch.add(onErrorCallable);
            // FIXME:  bug?  "processed" usually means consumer callback has completed; registerErrorEventsProcessed() normally called only if consumer callback returns ERROR:  why no registerErrorEventsReceived() call here?
            if (_consumerStats != null)
                _consumerStats.registerErrorEventsProcessed(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> onErrorCallable = _callbackFactory.createOnErrorCallable(curNanos, err, _loggingConsumer, false);
        _currentBatch.add(onErrorCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onError = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, true, true);
}
Also used : ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer)

Example 24 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onCheckpoint.

@Override
public ConsumerCallbackResult onCheckpoint(SCN checkpointScn) {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> checkpointCallable = _callbackFactory.createCheckpointCallable(curNanos, checkpointScn, consumer, true);
            _currentBatch.add(checkpointCallable);
            // really "callbacks received"
            if (_consumerStats != null)
                _consumerStats.registerEventsReceived(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> checkpointCallable = _callbackFactory.createCheckpointCallable(curNanos, checkpointScn, _loggingConsumer, false);
        _currentBatch.add(checkpointCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onCheckpoint = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, true, true);
}
Also used : ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer)

Example 25 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onStartDataEventSequence.

@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> startWindowCallable = _callbackFactory.createStartDataEventSequenceCallable(curNanos, startScn, consumer, true);
            _currentBatch.add(startWindowCallable);
            if (_consumerStats != null)
                _consumerStats.registerEventsReceived(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> startWindowCallable = _callbackFactory.createStartDataEventSequenceCallable(curNanos, startScn, _loggingConsumer, false);
        _currentBatch.add(startWindowCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onStartDataEventSequence = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, true, true);
}
Also used : ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer)

Aggregations

DatabusCombinedConsumer (com.linkedin.databus.client.pub.DatabusCombinedConsumer)26 ArrayList (java.util.ArrayList)15 DatabusV2ConsumerRegistration (com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration)9 SelectingDatabusCombinedConsumer (com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer)9 ConsumerCallbackResult (com.linkedin.databus.client.pub.ConsumerCallbackResult)9 AbstractDatabusCombinedConsumer (com.linkedin.databus.client.consumer.AbstractDatabusCombinedConsumer)8 IdNamePair (com.linkedin.databus.core.util.IdNamePair)8 HashMap (java.util.HashMap)8 Test (org.testng.annotations.Test)8 DbusEvent (com.linkedin.databus.core.DbusEvent)7 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)7 DelegatingDatabusCombinedConsumer (com.linkedin.databus.client.consumer.DelegatingDatabusCombinedConsumer)6 MultiConsumerCallback (com.linkedin.databus.client.consumer.MultiConsumerCallback)6 StreamConsumerCallbackFactory (com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory)6 DatabusStreamConsumer (com.linkedin.databus.client.pub.DatabusStreamConsumer)6 Checkpoint (com.linkedin.databus.core.Checkpoint)6 UncaughtExceptionTrackingThread (com.linkedin.databus.core.util.UncaughtExceptionTrackingThread)6 RegisterResponseEntry (com.linkedin.databus2.core.container.request.RegisterResponseEntry)6 List (java.util.List)6 DbusEventAppender (com.linkedin.databus.core.test.DbusEventAppender)5