Examples with DbusEventGenerator com.linkedin.databus.core.test.DbusEventGenerator used on opensource projects

Example 36 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPathWithOneEvent.

/**
 * Testcase send the following sequence
 * 1 valid packets, EOW
 * @throws Exception
 */
@Test
public void testNoMissingEOWHappyPathWithOneEvent() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    dbusBuf.start(0);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(2, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 2; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 2; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 37 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorGC.

@SuppressWarnings("unused")
@Test
public /**
 * Verify that internal iterators are automatically released during GC
 */
void testInternalIteratorGC() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorGC");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 100000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    log.info("acquire empty iterator on empty buffers ");
    InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(15, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("acquire oterh iterator on empty buffers ");
    // never move this one
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    // move this one a few events
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    iter2.next();
    iter2.next();
    iter2.next();
    // move this one to the end
    InternalEventIterator iter3 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    while (iter3.hasNext()) iter3.next();
    log.info("clear up iterators and try to GC");
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 4);
    Assert.assertEquals(dbusBuf._rwLockProvider.getNumReaders(), 4);
    iter0 = null;
    iter1 = null;
    iter2 = null;
    iter3 = null;
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            System.gc();
            // force release of GC'ed iterators
            dbusBuf.untrackIterator(null);
            log.debug("number of open iterators: " + dbusBuf._busyIteratorPool.size());
            log.debug("number of read locks: " + dbusBuf._rwLockProvider.getNumReaders());
            return 0 == dbusBuf._busyIteratorPool.size() && 0 == dbusBuf._rwLockProvider.getNumReaders();
        }
    }, "waiting for iterators and locks to be gone", 15000, log);
}

Example 38 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorWrapSingleBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * a single ByteBuffer
 */
void testInternalIteratorWrapSingleBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorWrapSingleBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    for (int i = 0; i < 1100; ++i) {
        log.info("add first window iteration " + i);
        final DbusEventGenerator generator = new DbusEventGenerator(40 * i);
        final Vector<DbusEvent> events1 = new Vector<DbusEvent>();
        generator.generateEvents(6, 1, 120, 41, events1);
        injectEventsInBuffer(dbusBuf, events1, true);
        log.info("get a new iterator");
        InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
        Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
        Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
        Assert.assertTrue(iter1.hasNext());
        log.info("process the last window");
        readAndCompareIteratorEvents(iter1, events1, true);
        iter1.close();
    }
    log.info("done");
}

Example 39 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferStreamEvents method testStreamLargeScn.

@Test
public /**
 * Tests streamEvents with a large scn with concurrent writes
 */
void testStreamLargeScn() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testStreamLargeScn");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(120000, 500000, 1024, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    final Checkpoint cp = Checkpoint.createOnlineConsumptionCheckpoint(1000);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final WritableByteChannel ochannel = Channels.newChannel(baos);
    final AtomicBoolean hasError = new AtomicBoolean(false);
    final AtomicInteger num = new AtomicInteger(0);
    final AtomicBoolean genComplete = new AtomicBoolean(false);
    UncaughtExceptionTrackingThread streamThread = new UncaughtExceptionTrackingThread(new Runnable() {

        @Override
        public void run() {
            try {
                while (!genComplete.get() && 0 >= num.get()) {
                    StreamEventsArgs args = new StreamEventsArgs(10000);
                    num.set(dbusBuf.streamEvents(cp, ochannel, args).getNumEventsStreamed());
                }
            } catch (ScnNotFoundException e) {
                hasError.set(true);
            } catch (OffsetNotFoundException e) {
                hasError.set(true);
            }
        }
    }, "testGetLargeScn.streamThread");
    streamThread.setDaemon(true);
    streamThread.start();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(70000, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    genComplete.set(true);
    streamThread.join(10000);
    Assert.assertFalse(streamThread.isAlive());
    Assert.assertNull(streamThread.getLastException());
    Assert.assertFalse(hasError.get());
    Assert.assertTrue(num.get() > 0);
}

Example 40 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testBasicRollback.

// Add a full window plus one event to the buffer, leaving room at the end, and then roll back.
// The write pointer should come back to the end of the event.
@Test
public void testBasicRollback() throws Exception {
    // We have 7 byte buffers now, the last one having 200 bytes, others with 500 bytes.
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(3200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, false);
    events.clear();
    // Same test case as before, with the first 3 byte buffers having data now.
    // total event size = 499
    generator.generateEvents(1, 20, 500, 438, events);
    // total event size = 71
    generator.generateEvents(1, 20, 500, 10, events);
    // total event size = 428
    generator.generateEvents(1, 20, 500, 367, events);
    // event + EOP = 132.
    generator.generateEvents(1, 1, 500, 10, events);
    appender.run();
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();
    // Now we should be able to add an event and roll it back as well.
    events.clear();
    // event size = 71, can append.
    generator.generateEvents(1, 20, 500, 10, events);
    long cwp1 = reflector.getCurrentWritePosition().getPosition();
    dbusBuf.startEvents();
    int evCount = 0;
    evCount = appender.addEventToBuffer(events.firstElement(), evCount);
    Assert.assertEquals(1, evCount);
    long cwp2 = reflector.getCurrentWritePosition().getPosition();
    Assert.assertTrue("Cwp should differ event write", cwp1 != cwp2);
    // Roll back.
    dbusBuf.rollbackEvents();
    Assert.assertEquals(cwp1, reflector.getCurrentWritePosition().getPosition());
}