Examples with BufferPositionParser com.linkedin.databus.core.util.BufferPositionParser used on opensource projects

Example 6 with BufferPositionParser

use of com.linkedin.databus.core.util.BufferPositionParser in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendBigEventsHeadScnOverlap.

@Test
public /**
   * Test the case where we have a big event that overlaps both the DbusEventBuffer head and the
   * limit of the current buffer.
   */
void testAppendBigEventsHeadScnOverlap() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendBigEventsHeadScnOverlap");
    log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(7, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(222, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(0, tailIndexId);
    assertEquals(61, tailOffset);
    log.info("append windows with one small and one big event");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    generator.generateEvents(1, 1, 280, 139, events);
    generator.generateEvents(1, 1, 480, 339, events);
    // Add events to the EventBuffer. Now the buffer is full
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(2, headIndexId);
    assertEquals(61, headOffset);
    assertEquals(1, tailIndexId);
    assertEquals(461, tailOffset);
    assertEquals(322, dbusBuf.getBuffer()[0].limit());
    log.info("finished");
}

Example 7 with BufferPositionParser

use of com.linkedin.databus.core.util.BufferPositionParser in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendEventWhenLimitLessThanCapacity.

@Test
public /**
   * Test the following case:
   * A buffer (one of 3) has CWP = 222 and head at 383. Limit is 483 (capacity 500)
   * we create one event of size 211 (61 + 150 payload).
   * we call appender.run() which will add one EOP event (61) and the newly created event 211.
   * The end of event offset will be 222 + 211 + 61 = 494. It should update limit to this value.
   * because of the bug (DDSDBUS-1515) it used to fail when end of event goes beyond current limit
   * but less than the capacity
   * Now it should pass
   */
void testAppendEventWhenLimitLessThanCapacity() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventWhenLimitLessThanCapacity");
    log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(11, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    // current writing position should be 222 (id=1)
    assertEquals(0, headGenId);
    assertEquals(1, headIndexId);
    assertEquals(383, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(1, tailIndexId);
    assertEquals(222, tailOffset);
    log.info("append event to stretch beyond limit but less than capacity");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    // will add two event 61 + 150
    generator.generateEvents(1, 1, 400, 150, events);
    // Add events. this will cause limit increased
    appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    assertEquals(1, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(61, headOffset);
    assertEquals(2, tailIndexId);
    assertEquals(61, tailOffset);
    assertEquals(494, dbusBuf.getBuffer()[1].limit());
    log.info("finished");
}

Example 8 with BufferPositionParser

use of com.linkedin.databus.core.util.BufferPositionParser in project databus by linkedin.

the class ReadEventsTestParams method testAppendEventOverlapNeq0.

/*
   * This test-case is to recreate the bug where appendEvents incorrectly writes event without
   *  moving head.
   *
   *  The following is the issue.
   *
   *  head is at location x and tail is at location y
   *  The next event window is such that after adding "some" number of events in the window, both head
   *  and currentWritePosition is same (differing only in genId). Now adding the next event corrupts the
   *  buffer (and scnIndex) as head is not moved ahead.
   *
   */
/**
   * Recreate the head and tail position such that the eventBuffer is in the below state:
   *
   * <pre>
   *                       NETBW
   *                      |-----|
   * ------------------------------------------------------------
   * ^      ^             ^                                     ^
   * |      |             |                                     |
   * 0      tail       CWP,head                                 capacity
   *
   * CWP : Current Write Position
   * NETBW : Next Event to be written
   * </pre>
   *
   * In this case, all pointers except head will be at (n+1)th generation while head
   * is at nth generation.
   *
   * Two test cases are covered here:
   *   1. n = 0
   *   2. n > 0
   */
@Test
public void testAppendEventOverlapNeq0() throws Exception // Case n = 0;
{
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(9, 3, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    //Logger.getRootLogger().setLevel(Level.ALL);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long scnIndexHead = dbusBuf.getScnIndex().getHead();
    long scnIndexTail = dbusBuf.getScnIndex().getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 0, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 1144, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 80, scnIndexTail);
    LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(100);
    /*
     * The event size is carefully created such that after adding 2nd
     * event CWP and tail points to the same location. Now the 3rd event corrupts the EVB and index (in the presence of bug).
     */
    generator.generateEvents(3, 2, 150, 89, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 783, headOffset);
    assertEquals("Tail GenId", 1, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 633, tailOffset);
    assertEquals("SCNIndex Head", 64, scnIndexHead);
    assertEquals("SCNIndex Tail", 48, scnIndexTail);
}

Example 9 with BufferPositionParser

use of com.linkedin.databus.core.util.BufferPositionParser in project databus by linkedin.

the class ReadEventsTestParams method testAppendEventOverlapMany.

//Case where we dump lot of events while reaching the error case many times during this process.
@Test
public void testAppendEventOverlapMany() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(9, 3, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long scnIndexHead = dbusBuf.getScnIndex().getHead();
    long scnIndexTail = dbusBuf.getScnIndex().getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 0, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 1144, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 80, scnIndexTail);
    LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
    /*
     * Dump lots of events
     */
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(100);
    generator.generateEvents(655, 3, 150, 89, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 109, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 511, headOffset);
    assertEquals("Tail GenId", 110, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 211, tailOffset);
    assertEquals("SCNIndex Head", 32, scnIndexHead);
    assertEquals("SCNIndex Tail", 16, scnIndexTail);
    /*
     * The event size is carefully created such that after adding 2nd
     * event CWP and tail points to the same location.
     */
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(10000);
    generator.generateEvents(3, 5, 100, 28, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(10000);
    generator.generateEvents(3, 3, 120, 19, events);
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 110, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 110, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 600, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 32, scnIndexTail);
}