Examples with DbusEventBuffer com.linkedin.databus.core.DbusEventBuffer used on opensource projects

Example 41 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestMultiConsumerCallback method test3GroupedStreamConsumersWithFailure.

@Test(groups = { "small", "functional" })
public void test3GroupedStreamConsumersWithFailure() {
    LOG.info("\n\nstarting test3GroupedStreamConsumersWithFailure()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusStreamConsumer mockConsumer2 = EasyMock.createStrictMock("consumer2", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer2, true);
    DatabusStreamConsumer mockConsumer3 = EasyMock.createStrictMock("consumer3", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer3, true);
    DatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    DatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    DatabusCombinedConsumer sdccMockConsumer3 = new SelectingDatabusCombinedConsumer(mockConsumer3);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg1 = new DatabusV2ConsumerRegistration(Arrays.asList(sdccMockConsumer1, sdccMockConsumer2, sdccMockConsumer3), sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg1);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockFailingStreamConsumer3OptEventFullLifecycle(mockConsumer1, event1, event2, event3, keyCounts);
    initMockFailingStreamConsumer3OptEventFullLifecycle(mockConsumer2, event1, event2, event3, keyCounts);
    initMockFailingStreamConsumer3OptEventFullLifecycle(mockConsumer3, event1, event2, event3, keyCounts);
    assert3EventFullLifecycleWithFailure(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer1);
    EasyMock.verify(mockConsumer2);
    EasyMock.verify(mockConsumer3);
    assert (keyCounts.get(1L).get() + keyCounts.get(2L).get() + keyCounts.get(3L).get()) == 3 : "invalid number of calls: " + keyCounts.get(1L).get() + "," + keyCounts.get(2L).get() + "," + keyCounts.get(3L).get();
}

Example 42 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestMultiConsumerCallback method testPerf.

@Test
public void testPerf() throws Exception {
    LOG.info("\n\nstarting testPerf()");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    iter.next();
    DbusEvent event1 = iter.next();
    DatabusStreamConsumer logConsumer = new LoggingConsumer();
    SelectingDatabusCombinedConsumer sdccLogConsumer = new SelectingDatabusCombinedConsumer(logConsumer);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccLogConsumer, sources, null);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newCachedThreadPool();
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, executor, 60000, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, null, null);
    callback.setSourceMap(sourcesMap);
    callback.onStartConsumption();
    callback.onStartDataEventSequence(new SingleSourceSCN(1, 1));
    for (int i = 0; i < 10000; ++i) {
        callback.onDataEvent(event1, null);
    }
    callback.onEndDataEventSequence(new SingleSourceSCN(1, 1));
    callback.onStopConsumption();
    System.out.println("max threads=" + executor.getLargestPoolSize() + " task count=" + executor.getTaskCount());
    System.out.println("dataEventsReceived=" + consumerStatsCollector.getNumDataEventsReceived() + " sysEventsReceived=" + consumerStatsCollector.getNumSysEventsReceived() + " dataEventsProcessed=" + consumerStatsCollector.getNumDataEventsProcessed() + " latencyEventsProcessed=" + consumerStatsCollector.getLatencyEventsProcessed());
    long dataEvents = consumerStatsCollector.getNumDataEventsReceived();
    assert (consumerStatsCollector.getNumDataEventsProcessed() == dataEvents);
}

Example 43 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testServerPartialTimeout.

@Test
public void testServerPartialTimeout() throws IOException, ScnNotFoundException, OffsetNotFoundException {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testServerPartialResponse");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testHappyPath");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    try {
        runServerPartialStreamTimeoutIteration(log, buf, callback, remoteExceptionHandler, conn);
    } finally {
        conn.close();
        callback.shutdown();
    }
}

Example 44 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method createSimpleBuffer.

private DbusEventBuffer createSimpleBuffer() {
    DbusEventBuffer buf = new DbusEventBuffer(_bufCfg);
    buf.start(0);
    buf.startEvents();
    buf.appendEvent(new DbusEventKey(1), (short) 1, (short) 1, System.nanoTime(), (short) 1, new byte[16], new byte[100], false, null);
    buf.appendEvent(new DbusEventKey(2), (short) 1, (short) 1, System.nanoTime(), (short) 1, new byte[16], new byte[100], false, null);
    buf.endEvents(10);
    return buf;
}

Example 45 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testServerFixedScenario.

/**
   * <p>Test scenario
   * <p>
   *
   * <ul>
   *   <li> relay disconnected on /sources
   *   <li> relay timed out on /sources
   *   <li> relay disconnected on /sources again
   *   <li> relay timed out on /register
   *   <li> relay disconnected on /register
   *   <li> relay timed out on /register again
   *   <li> relay disconnected on /stream
   *   <li> relay disconnected on /stream again
   *   <li> relay succeeded on /stream
   *   <li> relay succeeded on /stream
   *   <li> relay timed out on partial /stream
   *   <li> relay succeeded on /stream
   *   <li> relay timed out on /stream
   *   <li> relay succeeded on /stream
   * </ul>
   */
@Test
public void testServerFixedScenario() throws IOException, ScnNotFoundException, OffsetNotFoundException {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testServerFixedScenario");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testHappyPath");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    conn.getHandler().getLog().setLevel(_logLevel);
    try {
        log.info("********* 1. relay disconnected on /sources ********");
        runServerSourcesDisconnectIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 2. relay timed out on /sources ********");
        runServerSourcesReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 3. relay disconnected on /sources ********");
        runServerSourcesDisconnectIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 4. relay timed out on /register ********");
        runServerRegisterReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 5. relay disconnected on /register ********");
        runServerRegisterDisconnectIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 6. relay timed out on /register ********");
        runServerRegisterReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 7. relay disconnected on /stream ********");
        runServerStreamDisconnectIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 8. relay disconnected on /stream ********");
        runServerStreamDisconnectIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 9. relay success on /stream ********");
        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
        log.info("********* 10. relay success on /stream ********");
        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
        log.info("********* 11. relay timed out on partial /stream ********");
        runServerPartialStreamTimeoutIteration(log, buf, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 12. relay success on /stream ********");
        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
        log.info("********* 13. relay timed out on /stream ********");
        runServerStreamReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
        conn.getHandler().reset();
        log.info("********* 14. relay success on /stream ********");
        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
    } finally {
        conn.close();
        callback.shutdown();
    }
}