Examples with BufferPosition com.linkedin.databus.core.util.BufferPosition used on opensource projects

Example 1 with BufferPosition

use of com.linkedin.databus.core.util.BufferPosition in project databus by linkedin.

the class DbusEventBufferReflector method setBufferPositionField.

private BufferPosition setBufferPositionField(String fieldName) throws NoSuchFieldException, IllegalAccessException {
    Field field = DbusEventBuffer.class.getDeclaredField(fieldName);
    field.setAccessible(true);
    return (BufferPosition) field.get(_evb);
}

Example 2 with BufferPosition

use of com.linkedin.databus.core.util.BufferPosition in project databus by linkedin.

the class ReadEventsTestParams method testReadEventsAssertSpanError.

/**
   * Test to reproduce DDSDBUS-388.
   */
@Test
public void testReadEventsAssertSpanError() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testReadEventsAssertSpanError");
    //log.setLevel(Level.INFO);
    log.info("start");
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(10000, 10000, 320, 5000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    //dbusBuf.getLog().setLevel(Level.DEBUG);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    //event size:  10 + 61 = 71
    //window size:  5 * 71 + 61 = 416
    //layout: [61 + 23 * 416 + 5 * 71 = 9984 ] <WrapAround> [ 61 + 22 * 416 + 2 * 71 + 61 = 9416 ]
    //      [ H=Gen:0,Ofs:0,    T=Gen:0,Ofs:9984, L=10000 ]
    //      [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9416, L=9984 ]
    generator.generateEvents(232, 5, 100, 10, events);
    // Add events to the EventBuffer
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    log.info("Dbus Event Buffer is :" + dbusBuf);
    log.info("SCNIndex is :" + dbusBuf.getScnIndex());
    final BufferPosition headPos = new BufferPosition(dbusBuf.getHead(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    final BufferPosition tailPos = new BufferPosition(dbusBuf.getTail(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    Assert.assertEquals(0, headPos.bufferGenId());
    Assert.assertEquals(0, headPos.bufferIndex());
    Assert.assertEquals(9629, headPos.bufferOffset());
    Assert.assertEquals(1, tailPos.bufferGenId());
    Assert.assertEquals(0, tailPos.bufferIndex());
    Assert.assertEquals(9416, tailPos.bufferOffset());
    Assert.assertEquals(9984, dbusBuf.getBuffer()[0].limit());
    dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
    final BufferPosition lastWinStart = new BufferPosition(61 + 22 * 416, dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
    long lastScn = events.get(events.size() - 1).sequence();
    generator = new DbusEventGenerator(lastScn - 1);
    events.clear();
    //event size: 10 + 61 = 71
    generator.generateEvents(2, 2, 75, 10, events);
    //intended layout: append 2 * 71  = 142 bytes
    //      [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9558, L=9984 ]
    lastScn = events.get(events.size() - 1).sequence();
    generator = new DbusEventGenerator(lastScn - 1);
    //event size: 500 + 61 = 561
    //partial window size: 561  = 622
    generator.generateEvents(1, 1, 800, 500, events);
    //intended layout: append 561 bytes;
    //      [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
    // head moves to first window after 561: 61 + 2 * 416 = 893 > 561
    //Set up the ReadChannel with new events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (DbusEvent e : events) {
        assertTrue("invalid event", e.isValid());
        ((DbusEventInternalReadable) e).writeTo(oChannel, Encoding.BINARY);
        log.debug("event size is: " + e.size());
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    dbusBuf.readEvents(rChannel);
    log.info("Dbus Event Buffer is :" + dbusBuf);
    log.info("SCNIndex is :" + dbusBuf.getScnIndex());
    //expected layout:
    //      [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
    headPos.setPosition(dbusBuf.getHead());
    Assert.assertEquals(1, headPos.bufferGenId());
    Assert.assertEquals(0, headPos.bufferIndex());
    Assert.assertEquals(893, headPos.bufferOffset());
    tailPos.setPosition(dbusBuf.getTail());
    Assert.assertEquals(2, tailPos.bufferGenId());
    Assert.assertEquals(0, tailPos.bufferIndex());
    Assert.assertEquals(561, tailPos.bufferOffset());
    Assert.assertEquals(9558, dbusBuf.getBuffer()[0].limit());
    dbusBuf.getBufferPositionParser().assertSpan(dbusBuf.getHead(), dbusBuf.getTail(), true);
    dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
    //the second write did not add EOP so the SCN index tail should not have changed
    dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
}

Example 3 with BufferPosition

use of com.linkedin.databus.core.util.BufferPosition in project databus by linkedin.

the class DbusEventBuffer method ensureFreeSpace.

/**
   * Makes sure that we have enough space at the destination write buffer. Depending on the
   * queuing policy, we can either wait for space to free up or will overwrite it.
   *
   * @param writeStartPos     the gen-id starting write position
   * @param writeEndPos       the gen-id ending write position (after the last byte written)
   * @param logDebugEnabled   a flag if debug logging messages are enabled
   * @return true if the wait for free space was interrupted prematurely
   */
private boolean ensureFreeSpace(long writeStartPos, long writeEndPos, boolean logDebugEnabled) {
    // Normalize the write end position to make sure that if we have to move
    // the head, it points to real data.  This deals with two code-path
    // variants of the same basic bug.  Specifically, if the head points at the
    // current bytebuffer's limit (i.e., at invalid data at the end of it), and
    // the proposed end position is exactly equal to that, BufferPositionParser's
    // incrementOffset() (a.k.a. sanitize()) advances the position to the start
    // of the next buffer.  Since this is beyond (or "overruns") the head, the
    // subsequent call to setNextFreePos(), which calls moveCurrentWritePosition(),
    // blows up.  By normalizing the end position, we effectively block until the
    // head can advance to the same position (or beyond), i.e., until all iterators
    // have caught up, which allows setNextFreePos()/moveCurrentWritePosition() to
    // succeed.  See DDSDBUS-1816 for even more details.
    final BufferPosition normalizedWriteEndPos = new BufferPosition(writeEndPos, _bufferPositionParser, _buffers);
    normalizedWriteEndPos.skipOverFreeSpace();
    boolean interrupted = false;
    if (!empty()) {
        if (QueuePolicy.BLOCK_ON_WRITE == _queueingPolicy) {
            interrupted = waitForReadEventsFreeSpace(logDebugEnabled, writeStartPos, normalizedWriteEndPos.getPosition());
        } else {
            freeUpSpaceForReadEvents(logDebugEnabled, writeStartPos, normalizedWriteEndPos.getPosition());
        }
    }
    if (logDebugEnabled) {
        _log.debug("ensureFreeSpace: writeStart:" + _bufferPositionParser.toString(writeStartPos, _buffers) + "; writeEnd:" + _bufferPositionParser.toString(writeEndPos, _buffers) + "; normalizedWriteEnd:" + normalizedWriteEndPos + "; head:" + _head + "; tail:" + _tail + "; interrupted:" + interrupted);
    }
    assert interrupted || !overwritesHead(writeStartPos, normalizedWriteEndPos.getPosition());
    return interrupted;
}