Examples with DbusEventBufferReflector com.linkedin.databus.core.test.DbusEventBufferReflector used on opensource projects

Example 1 with DbusEventBufferReflector

use of com.linkedin.databus.core.test.DbusEventBufferReflector in project databus by linkedin.

the class ReadEventsTestParams method testMaxEventSize.

@Test
public void testMaxEventSize() throws Exception {
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1144, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    DbusEventGenerator generator = new DbusEventGenerator();
    // Generate one event that equals the size of the first buffer. We won't be able to
    // append that to the buffer.
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(1, 2, 500, 439, events);
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, false);
    int eventCount = 0;
    boolean exceptionCaught = false;
    dbusBuf.startEvents();
    try {
        eventCount = appender.addEventToBuffer(events.get(0), eventCount);
    } catch (DatabusRuntimeException e) {
        exceptionCaught = true;
    }
    Assert.assertTrue(exceptionCaught);
    Assert.assertEquals(0, eventCount);
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();
    Assert.assertEquals(0, reflector.getBuffer(0).limit());
    Assert.assertEquals(0, reflector.getBuffer(1).limit());
    Assert.assertEquals(0, reflector.getBuffer(2).limit());
    events.clear();
    dbusBuf = new DbusEventBuffer(getConfig(1144, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    appender = new DbusEventAppender(events, dbusBuf, null, false);
    // total event size = 499
    generator.generateEvents(1, 20, 500, 438, events);
    // total event size = 71
    generator.generateEvents(1, 20, 500, 10, events);
    // total event size = 428
    generator.generateEvents(1, 20, 500, 367, events);
    // event + EOP = 132.
    generator.generateEvents(1, 1, 500, 10, events);
    // We should be able to append the first three events above, filling the two byte buffers completely.
    // And then the last event along with EOP marker in the last byte buffer, filling that to complete
    // one window.
    appender.run();
    // Now try to add one event. Because ScnIndex still has head as -1 for this buffer,
    // we end up getting a DatabusRuntimeException and not being able to add an event.
    // Since this is a rare case, it is TBD whether we need to support this case or not.
    events.clear();
    // event size = 71, can append.
    generator.generateEvents(1, 20, 500, 10, events);
    exceptionCaught = false;
    try {
        appender.run();
    } catch (DatabusRuntimeException e) {
        exceptionCaught = true;
    }
    Assert.assertTrue(exceptionCaught);
}

Example 2 with DbusEventBufferReflector

use of com.linkedin.databus.core.test.DbusEventBufferReflector in project databus by linkedin.

the class ReadEventsTestParams method testCompleteRollback.

// Add a bunch of events to the buffer without EOP, and then issue a rollback.
// Verify that the buffer is now empty, and we can add the same events with EOP again.
@Test
public void testCompleteRollback() throws Exception {
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1144, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, false);
    events.clear();
    // Same test case as before, with the first 3 byte buffers having data now.
    // total event size = 499
    generator.generateEvents(1, 20, 500, 438, events);
    // total event size = 71
    generator.generateEvents(1, 20, 500, 10, events);
    // total event size = 428
    generator.generateEvents(1, 20, 500, 367, events);
    // event = 71.
    generator.generateEvents(1, 20, 500, 10, events);
    dbusBuf.startEvents();
    int evCount = 0;
    for (DbusEvent e : events) {
        evCount = appender.addEventToBuffer(e, evCount);
    }
    Assert.assertEquals(events.size(), evCount);
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();
    dbusBuf.rollbackEvents();
    Assert.assertEquals(0, reflector.getCurrentWritePosition().getPosition());
    // Now we should be able to add these events, plus an EOP marker at the end.
    appender.run();
    long cwp1 = reflector.getCurrentWritePosition().getPosition();
    Assert.assertEquals(1156, cwp1);
    long tail = reflector.getTail().getPosition();
    Assert.assertEquals(1156, tail);
    Assert.assertEquals(0, reflector.getHead().getPosition());
}

Example 3 with DbusEventBufferReflector

use of com.linkedin.databus.core.test.DbusEventBufferReflector in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendEventsWithCloseInMiddle.

@Test
public /**
   * verify that changes to the buffer (appendEvent, endEvent, clear) after the buffer is closed
   * are ignored (and rolled back)
   * we do this by creating buffer (or loading from mmap files from the previous run), adding one window,
   * then doing append, close , append/endEvents (which should throw an exception)
   * then loading again and verifying that it has our events
   */
void testAppendEventsWithCloseInMiddle() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventsWithCloseInMiddle");
    log.setLevel(Level.DEBUG);
    log.info("starting");
    // mmap dir
    File mmapDir = initMmapDir("/tmp/tmp_mmapDir");
    DbusEventBuffer.StaticConfig config = getConfig(1200, 500, 100, 500, AllocationPolicy.MMAPPED_MEMORY, mmapDir.getAbsolutePath(), true);
    DbusEventBuffer dbusBuf = new DbusEventBuffer(config);
    DbusEventBufferReflector bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(12, 2, 120, 39, events);
    // add first window with 2 events
    dbusBuf.start(-1);
    int i = 0;
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // add first event of the second window
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // now add the end
    try {
        addOneEvent(events.get(i), dbusBuf, EventType.END);
    } catch (Throwable ex) {
        log.info("Got e: ", ex);
    }
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // create new buffer
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // add two more events but don't do endEvents
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // no endEvents()
    addOneEvent(events.get(i++), dbusBuf, EventType.REG);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // should be ok (WARN in the logs)
    dbusBuf.closeBuffer(true);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // call endEvents(on a closed buffer);
    try {
        dbusBuf.endEvents(events.get(i - 1).sequence());
    } catch (Throwable ex) {
        log.info("Got e2: ", ex);
    }
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // create new buffer
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // add two more events but don't do endEvents
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // no endEvents()
    addOneEvent(events.get(i++), dbusBuf, EventType.REG);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // call endEvents(on a closed buffer);
    try {
        //should fail
        dbusBuf.clear();
    } catch (Throwable ex) {
        log.info("Got e3: ", ex);
    }
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // make sure it is still valid
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
}

Example 4 with DbusEventBufferReflector

use of com.linkedin.databus.core.test.DbusEventBufferReflector in project databus by linkedin.

the class ReadEventsTestParams method testBasicRollback.

// Add a full window plus one event to the buffer, leaving room at the end, and then roll back.
// The write pointer should come back to the end of the event.
@Test
public void testBasicRollback() throws Exception {
    // We have 7 byte buffers now, the last one having 200 bytes, others with 500 bytes.
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(3200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, false);
    events.clear();
    // Same test case as before, with the first 3 byte buffers having data now.
    // total event size = 499
    generator.generateEvents(1, 20, 500, 438, events);
    // total event size = 71
    generator.generateEvents(1, 20, 500, 10, events);
    // total event size = 428
    generator.generateEvents(1, 20, 500, 367, events);
    // event + EOP = 132.
    generator.generateEvents(1, 1, 500, 10, events);
    appender.run();
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();
    // Now we should be able to add an event and roll it back as well.
    events.clear();
    // event size = 71, can append.
    generator.generateEvents(1, 20, 500, 10, events);
    long cwp1 = reflector.getCurrentWritePosition().getPosition();
    dbusBuf.startEvents();
    int evCount = 0;
    evCount = appender.addEventToBuffer(events.firstElement(), evCount);
    Assert.assertEquals(1, evCount);
    long cwp2 = reflector.getCurrentWritePosition().getPosition();
    Assert.assertTrue("Cwp should differ event write", cwp1 != cwp2);
    // Roll back.
    dbusBuf.rollbackEvents();
    Assert.assertEquals(cwp1, reflector.getCurrentWritePosition().getPosition());
}