Examples with DbusEventGenerator com.linkedin.databus.core.test.DbusEventGenerator used on opensource projects

Example 11 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class ReadEventsTestParams method runConstEventsReaderWriter.

// data flow:  emitter/producer/appender -> DEB -> writer -> pipe -> reader -> DEB -> consumer
protected boolean runConstEventsReaderWriter(Vector<DbusEvent> srcTestEvents, Vector<DbusEvent> dstTestEvents, EventBufferTestInput input, DbusEventsStatisticsCollector emitterStats, DbusEventsStatisticsCollector streamStats, DbusEventsStatisticsCollector clientStats, boolean autoStartBuffer) throws Exception {
    LOG.info("starting runConstEventsReaderWriter for " + input.getTestName());
    int numEvents = input.getNumEvents();
    int maxWindowSize = input.getWindowSize();
    DbusEventGenerator evGen = new DbusEventGenerator();
    if (evGen.generateEvents(numEvents, maxWindowSize, 512, input.getPayloadSize(), srcTestEvents) <= 0) {
        return false;
    }
    int eventSize = srcTestEvents.get(0).size();
    long producerBufferSize = input.getProducerBufferSize() * eventSize;
    long sharedBufferSize = input.getSharedBufferSize() * eventSize;
    int stagingBufferSize = input.getStagingBufferSize() * eventSize;
    int individualBufferSize = input.getIndividualBufferSize() * eventSize;
    int indexSize = input.getIndexSize() * eventSize;
    QueuePolicy prodQueuePolicy = input.getProdQueuePolicy();
    QueuePolicy consQueuePolicy = input.getConsQueuePolicy();
    // create the main event buffers
    DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, prodQueuePolicy, input.getProdBufferAssertLevel()));
    DbusEventBuffer consEventBuffer = new DbusEventBuffer(getConfig(sharedBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, consQueuePolicy, input.getConsBufferAssertLevel()));
    // Producer of events, a.k.a. "emitter"
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats, autoStartBuffer);
    // commn channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    readerStream.configureBlocking(false);
    // Event writer - Relay in the real world
    int batchSize = input.getBatchSize() * eventSize;
    DbusEventBufferWriter writer = new DbusEventBufferWriter(prodEventBuffer, writerStream, batchSize, streamStats);
    // Event readers - Clients in the real world
    DbusEventBufferConsumer consumer = new DbusEventBufferConsumer(consEventBuffer, numEvents, input.getDeleteInterval(), dstTestEvents);
    Vector<EventBufferConsumer> consList = new Vector<EventBufferConsumer>();
    consList.add(consumer);
    // Event readers - Clients in the real world
    DbusEventBufferReader reader = new DbusEventBufferReader(consEventBuffer, readerStream, consList, clientStats);
    UncaughtExceptionTrackingThread tEmitter = new UncaughtExceptionTrackingThread(eventProducer, "EventProducer");
    UncaughtExceptionTrackingThread tWriter = new UncaughtExceptionTrackingThread(writer, "Writer");
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    UncaughtExceptionTrackingThread tConsumer = new UncaughtExceptionTrackingThread(consumer, "Consumer");
    long emitterWaitms = 20000;
    long writerWaitms = 10000;
    long readerWaitms = 10000;
    long consumerWaitms = readerWaitms;
    // start emitter;
    tEmitter.start();
    // tarnish events written to buffer;
    int[] corruptIndexList = input.getCorruptIndexList();
    if (corruptIndexList.length > 0) {
        tEmitter.join(emitterWaitms);
        EventCorruptionType corruptionType = input.getCorruptionType();
        eventProducer.tarnishEventsInBuffer(corruptIndexList, corruptionType);
    }
    // start  consumer / reader /writer
    tConsumer.start();
    tWriter.start();
    tReader.start();
    // wait until all events have been written;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): waiting up to " + (emitterWaitms / 1000) + " sec for appender/producer/emitter thread to terminate");
    tEmitter.join(emitterWaitms);
    // try and set a finish for writer
    long eventsEmitted = eventProducer.eventsEmitted();
    writer.setExpectedEvents(eventsEmitted);
    // wait for writer to finish;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): waiting up to " + (writerWaitms / 1000) + " sec for writer thread to terminate");
    tWriter.join(writerWaitms);
    // close the writer Stream;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): signalling writer to stop");
    writer.stop();
    if (!tReader.isAlive()) {
        LOG.error("runConstEventsReaderWriter(): reader thread died unexpectedly");
    }
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): waiting up to " + (consumerWaitms / 1000) + " sec for consumer thread to terminate");
    tConsumer.join(consumerWaitms);
    // stop the consumer thread; may or may not have got all events;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): signalling consumer to stop");
    consumer.stop();
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): signalling reader to stop");
    reader.stop();
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): all stop.");
    assertEquals(null, consumer.getExceptionThrown());
    LOG.info("runConstEventsReaderWriter() consumer thread: " + (consumer.hasInvalidEvent() ? "DID" : "did NOT") + " receive invalid event(s); num events emitted=" + eventsEmitted + ", events written=" + writer.eventsWritten() + ", events read=" + reader.eventsRead());
    return true;
}

Example 12 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorHappyPathMultiBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * multiple ByteBuffers
 */
void testInternalIteratorHappyPathMultiBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorHappyPathMultiBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 400, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.ALL));
    log.info("verify iterators on empty buffers ");
    final InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter0.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter0._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter0.getIdentifier());
    Assert.assertTrue(iter0.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(5, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("verify we can read all events");
    // old iterator has not changed
    Assert.assertTrue(!iter0.hasNext());
    log.info("verify new iterator");
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter1.getIdentifier());
    Assert.assertTrue(iter1.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter1.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter1, events, false);
    log.info("add more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    final DbusEventGenerator generator2 = new DbusEventGenerator(1000);
    generator2.generateEvents(50, 4, 180, 100, events2);
    injectEventsInBuffer(dbusBuf, events2, false);
    log.info("verify old iterators have not changed");
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertTrue(!iter1.hasNext());
    log.info("verify new iterator");
    events.addAll(events2);
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter2.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter2.getIdentifier());
    Assert.assertTrue(iter2.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 3);
    Assert.assertTrue(iter2.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter2, events, false);
    iter0.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    iter2.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    iter1.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 0);
    log.info("done");
}

Example 13 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferIterators method testInternalIteratorHappyPathSingleBuf.

@Test
public /**
 * Verify internal iterators see all current updates and nothing after that for a event buffer with
 * a single ByteBuffer
 */
void testInternalIteratorHappyPathSingleBuf() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferIterator.testInternalIteratorHappyPathSingleBuf");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(100000, 100000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    log.info("verify iterators on empty buffers ");
    final InternalEventIterator iter0 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter0.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter0._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter0.getIdentifier());
    Assert.assertTrue(iter0.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    log.info("append a full window of events");
    final DbusEventGenerator generator = new DbusEventGenerator();
    final Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(5, 5, 120, 39, events);
    injectEventsInBuffer(dbusBuf, events, false);
    log.info("verify we can read all events");
    // old iterator has not changed
    Assert.assertTrue(!iter0.hasNext());
    log.info("verify new iterator");
    InternalEventIterator iter1 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter1.getIdentifier());
    Assert.assertTrue(iter1.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter1.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter1, events, false);
    log.info("add more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    final DbusEventGenerator generator2 = new DbusEventGenerator(1000);
    generator2.generateEvents(50, 4, 180, 100, events2);
    injectEventsInBuffer(dbusBuf, events2, false);
    log.info("verify old iterators have not changed");
    Assert.assertTrue(!iter0.hasNext());
    Assert.assertTrue(!iter1.hasNext());
    log.info("verify new iterator");
    events.addAll(events2);
    InternalEventIterator iter2 = dbusBuf.acquireInternalIterator(dbusBuf.getHead(), dbusBuf.getTail(), null);
    Assert.assertEquals(iter2.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertNotNull(iter2.getIdentifier());
    Assert.assertTrue(iter2.getIdentifier().startsWith(InternalEventIterator.class.getSimpleName()));
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 3);
    Assert.assertTrue(iter2.hasNext());
    log.info("make sure we can read all events");
    readAndCompareIteratorEvents(iter2, events, false);
    iter0.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    iter2.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    iter1.close();
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 0);
    log.info("done");
}

Example 14 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendEventsWithCloseInMiddle.

@Test
public /**
 * verify that changes to the buffer (appendEvent, endEvent, clear) after the buffer is closed
 * are ignored (and rolled back)
 * we do this by creating buffer (or loading from mmap files from the previous run), adding one window,
 * then doing append, close , append/endEvents (which should throw an exception)
 * then loading again and verifying that it has our events
 */
void testAppendEventsWithCloseInMiddle() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventsWithCloseInMiddle");
    log.setLevel(Level.DEBUG);
    log.info("starting");
    // mmap dir
    File mmapDir = initMmapDir("/tmp/tmp_mmapDir");
    DbusEventBuffer.StaticConfig config = getConfig(1200, 500, 100, 500, AllocationPolicy.MMAPPED_MEMORY, mmapDir.getAbsolutePath(), true);
    DbusEventBuffer dbusBuf = new DbusEventBuffer(config);
    DbusEventBufferReflector bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(12, 2, 120, 39, events);
    // add first window with 2 events
    dbusBuf.start(-1);
    int i = 0;
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // add first event of the second window
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // now add the end
    try {
        addOneEvent(events.get(i), dbusBuf, EventType.END);
    } catch (Throwable ex) {
        log.info("Got e: ", ex);
    }
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    // create new buffer
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 2 + 1(EOW)
    checkEventsInBuffer(bufferReflector, 3);
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // add two more events but don't do endEvents
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // no endEvents()
    addOneEvent(events.get(i++), dbusBuf, EventType.REG);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // should be ok (WARN in the logs)
    dbusBuf.closeBuffer(true);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // call endEvents(on a closed buffer);
    try {
        dbusBuf.endEvents(events.get(i - 1).sequence());
    } catch (Throwable ex) {
        log.info("Got e2: ", ex);
    }
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    // create new buffer
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 4 + 2(EOW)
    checkEventsInBuffer(bufferReflector, 6);
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    addOneEvent(events.get(i++), dbusBuf, EventType.END);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // add two more events but don't do endEvents
    addOneEvent(events.get(i++), dbusBuf, EventType.START);
    // no endEvents()
    addOneEvent(events.get(i++), dbusBuf, EventType.REG);
    // now close the buffer
    dbusBuf.closeBuffer(true);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // call endEvents(on a closed buffer);
    try {
        // should fail
        dbusBuf.clear();
    } catch (Throwable ex) {
        log.info("Got e3: ", ex);
    }
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
    // make sure it is still valid
    // should load from mmap
    dbusBuf = new DbusEventBuffer(config);
    bufferReflector = new DbusEventBufferReflector(dbusBuf);
    // 6 + 3(EOW)
    checkEventsInBuffer(bufferReflector, 9);
}

Example 15 with DbusEventGenerator

use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendBigEventsHeadScnOverlap.

@Test
public /**
 * Test the case where we have a big event that overlaps both the DbusEventBuffer head and the
 * limit of the current buffer.
 */
void testAppendBigEventsHeadScnOverlap() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendBigEventsHeadScnOverlap");
    log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(7, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(222, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(0, tailIndexId);
    assertEquals(61, tailOffset);
    log.info("append windows with one small and one big event");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    generator.generateEvents(1, 1, 280, 139, events);
    generator.generateEvents(1, 1, 480, 339, events);
    // Add events to the EventBuffer. Now the buffer is full
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(2, headIndexId);
    assertEquals(61, headOffset);
    assertEquals(1, tailIndexId);
    assertEquals(461, tailOffset);
    assertEquals(322, dbusBuf.getBuffer()[0].limit());
    log.info("finished");
}