Examples with InternalEventIterator com.linkedin.databus.core.DbusEventBuffer.InternalEventIterator used on opensource projects

Example 1 with InternalEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.InternalEventIterator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorHappyPathMultiBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * multiple ByteBuffers
 */
void testInternalIteratorHappyPathMultiBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorHappyPathMultiBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 400, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.ALL));
    log.info("verify iterators on empty buffers ");
    final InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter0.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter0._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter0.getIdentifier());
    Assert.assertTrue(iter0.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(5, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("verify we can read all events");
    // old iterator has not changed
    Assert.assertTrue(!iter0.hasNext());
    log.info("verify new iterator");
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter1.getIdentifier());
    Assert.assertTrue(iter1.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter1.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter1, events, false);
    log.info("add more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    final DbusEventGenerator generator2 = new DbusEventGenerator(1000);
    generator2.generateEvents(50, 4, 180, 100, events2);
    injectEventsInBuffer(dbusBuf, events2, false);
    log.info("verify old iterators have not changed");
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertTrue(!iter1.hasNext());
    log.info("verify new iterator");
    events.addAll(events2);
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter2.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter2.getIdentifier());
    Assert.assertTrue(iter2.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 3);
    Assert.assertTrue(iter2.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter2, events, false);
    iter0.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    iter2.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    iter1.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 0);
    log.info("done");
}

Example 2 with InternalEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.InternalEventIterator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorHappyPathSingleBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * a single ByteBuffer
 */
void testInternalIteratorHappyPathSingleBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorHappyPathSingleBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 100000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    log.info("verify iterators on empty buffers ");
    final InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter0.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter0._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter0.getIdentifier());
    Assert.assertTrue(iter0.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(5, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("verify we can read all events");
    // old iterator has not changed
    Assert.assertTrue(!iter0.hasNext());
    log.info("verify new iterator");
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter1.getIdentifier());
    Assert.assertTrue(iter1.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter1.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter1, events, false);
    log.info("add more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    final DbusEventGenerator generator2 = new DbusEventGenerator(1000);
    generator2.generateEvents(50, 4, 180, 100, events2);
    injectEventsInBuffer(dbusBuf, events2, false);
    log.info("verify old iterators have not changed");
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertTrue(!iter1.hasNext());
    log.info("verify new iterator");
    events.addAll(events2);
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter2.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter2.getIdentifier());
    Assert.assertTrue(iter2.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 3);
    Assert.assertTrue(iter2.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter2, events, false);
    iter0.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    iter2.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    iter1.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 0);
    log.info("done");
}

Example 3 with InternalEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.InternalEventIterator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorGC.

@SuppressWarnings("unused")
@Test
public /**
 * Verify that internal iterators are automatically released during GC
 */
void testInternalIteratorGC() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorGC");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 100000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    log.info("acquire empty iterator on empty buffers ");
    InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(15, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("acquire oterh iterator on empty buffers ");
    // never move this one
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    // move this one a few events
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    iter2.next();
    iter2.next();
    iter2.next();
    // move this one to the end
    InternalEventIterator iter3 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    while (iter3.hasNext()) iter3.next();
    log.info("clear up iterators and try to GC");
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 4);
    Assert.assertEquals(dbusBuf._rwLockProvider.getNumReaders(), 4);
    iter0 = null;
    iter1 = null;
    iter2 = null;
    iter3 = null;
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            System.gc();
            // force release of GC'ed iterators
            dbusBuf.untrackIterator(null);
            log.debug("number of open iterators: " + dbusBuf._busyIteratorPool.size());
            log.debug("number of read locks: " + dbusBuf._rwLockProvider.getNumReaders());
            return 0 == dbusBuf._busyIteratorPool.size() && 0 == dbusBuf._rwLockProvider.getNumReaders();
        }
    }, "waiting for iterators and locks to be gone", 15000, log);
}

Example 4 with InternalEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.InternalEventIterator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorWrapSingleBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * a single ByteBuffer
 */
void testInternalIteratorWrapSingleBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorWrapSingleBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    for (int i = 0; i < 1100; ++i) {
        log.info("add first window iteration " + i);
        final DbusEventGenerator generator = new DbusEventGenerator(40 * i);
        final Vector<DbusEvent> events1 = new Vector<DbusEvent>();
        generator.generateEvents(6, 1, 120, 41, events1);
        injectEventsInBuffer(dbusBuf, events1, true);
        log.info("get a new iterator");
        InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
        Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
        Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
        Assert.assertTrue(iter1.hasNext());
        log.info("process the last window");
        readAndCompareIteratorEvents(iter1, events1, true);
        iter1.close();
    }
    log.info("done");
}