Examples with DbusEventGenerator com.linkedin.databus.core.test.DbusEventGenerator used on opensource projects

Example 31 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPath.

/**
 * Testcase sends events with the following sequence :
 * valid packet, EOW, valid packet
 */
@Test
public void testNoMissingEOWHappyPath() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(3, 1, 100, 10, events);
    // don't try this at home, kids!
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 3; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
        LOG.info("DbusEvent " + i + " " + e);
    }
    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 3; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 32 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testAppendEventOverlapNeq0.

/*
   * This test-case is to recreate the bug where appendEvents incorrectly writes event without
   *  moving head.
   *
   *  The following is the issue.
   *
   *  head is at location x and tail is at location y
   *  The next event window is such that after adding "some" number of events in the window, both head
   *  and currentWritePosition is same (differing only in genId). Now adding the next event corrupts the
   *  buffer (and scnIndex) as head is not moved ahead.
   *
   */
/**
 * Recreate the head and tail position such that the eventBuffer is in the below state:
 *
 * <pre>
 *                       NETBW
 *                      |-----|
 * ------------------------------------------------------------
 * ^      ^             ^                                     ^
 * |      |             |                                     |
 * 0      tail       CWP,head                                 capacity
 *
 * CWP : Current Write Position
 * NETBW : Next Event to be written
 * </pre>
 *
 * In this case, all pointers except head will be at (n+1)th generation while head
 * is at nth generation.
 *
 * Two test cases are covered here:
 *   1. n = 0
 *   2. n > 0
 */
@Test
public void testAppendEventOverlapNeq0() throws Exception // Case n = 0;
{
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(9, 3, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // Logger.getRootLogger().setLevel(Level.ALL);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long scnIndexHead = dbusBuf.getScnIndex().getHead();
    long scnIndexTail = dbusBuf.getScnIndex().getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 0, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 1144, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 80, scnIndexTail);
    LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(100);
    /*
     * The event size is carefully created such that after adding 2nd
     * event CWP and tail points to the same location. Now the 3rd event corrupts the EVB and index (in the presence of bug).
     */
    generator.generateEvents(3, 2, 150, 89, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 783, headOffset);
    assertEquals("Tail GenId", 1, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 633, tailOffset);
    assertEquals("SCNIndex Head", 64, scnIndexHead);
    assertEquals("SCNIndex Tail", 48, scnIndexTail);
}

Example 33 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferPersistence method pushEventsToBuffer.

private void pushEventsToBuffer(DbusEventBuffer dbusBuf, int numEvents) {
    dbusBuf.start(1);
    dbusBuf.startEvents();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(numEvents, 1, 100, 10, events);
    // set end of windows
    for (int i = 0; i < numEvents - 1; ++i) {
        long scn = events.get(i).sequence();
        ++i;
        DbusEventInternalWritable writableEvent;
        try {
            writableEvent = DbusEventCorrupter.makeWritable(events.get(i));
        } catch (InvalidEventException ie) {
            LOG.error("Exception trace is " + ie);
            Assert.fail();
            return;
        }
        writableEvent.setSrcId((short) -2);
        writableEvent.setSequence(scn);
        writableEvent.applyCrc();
        assertTrue("invalid event #" + i, writableEvent.isValid(true));
    }
    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < numEvents; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 34 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testAppendEventOverlapMany.

// Case where we dump lot of events while reaching the error case many times during this process.
@Test
public void testAppendEventOverlapMany() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(9, 3, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long scnIndexHead = dbusBuf.getScnIndex().getHead();
    long scnIndexTail = dbusBuf.getScnIndex().getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals("Head GenId", 0, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 0, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 1144, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 80, scnIndexTail);
    LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
    /*
     * Dump lots of events
     */
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(100);
    generator.generateEvents(655, 3, 150, 89, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 109, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 511, headOffset);
    assertEquals("Tail GenId", 110, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 211, tailOffset);
    assertEquals("SCNIndex Head", 32, scnIndexHead);
    assertEquals("SCNIndex Tail", 16, scnIndexTail);
    /*
     * The event size is carefully created such that after adding 2nd
     * event CWP and tail points to the same location.
     */
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(10000);
    generator.generateEvents(3, 5, 100, 28, events);
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    events = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(10000);
    generator.generateEvents(3, 3, 120, 19, events);
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    scnIndexHead = dbusBuf.getScnIndex().getHead();
    scnIndexTail = dbusBuf.getScnIndex().getTail();
    assertEquals("Head GenId", 110, headGenId);
    assertEquals("Head Index", 0, headIndexId);
    assertEquals("Head Offset", 0, headOffset);
    assertEquals("Tail GenId", 110, tailGenId);
    assertEquals("Tail Index", 0, tailIndexId);
    assertEquals("Tail Offset", 600, tailOffset);
    assertEquals("SCNIndex Head", 0, scnIndexHead);
    assertEquals("SCNIndex Tail", 32, scnIndexTail);
}

Example 35 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPathWithMultiEventWindowWithMultiEOWs.

/**
 * Testcase send the following sequence:
 * two valid packets, EOW, EOW, one valid packet, EOW
 * @throws Exception
 */
@Test
public void testNoMissingEOWHappyPathWithMultiEventWindowWithMultiEOWs() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(6, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(3));
    writableEvent.setSequence(events.get(0).sequence() + 50);
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(4));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(5));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 6; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 6; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie.getMessage(), ie);
        Assert.fail();
        return;
    }
}