Examples with DbusEventAppender com.linkedin.databus.core.test.DbusEventAppender used on opensource projects

Example 11 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestDbusEventBufferIterators method injectEventsInBuffer.

/**
 * Write the events directly to the ByteBuffer because we want to avoid invoking internal
 * listeners which in turn will try to use iterators. A bit of a Catch-22.
 */
private static void injectEventsInBuffer(DbusEventBuffer buf, Vector<DbusEvent> events, boolean updateScnIndex) throws Exception {
    DbusEventAppender appender = new DbusEventAppender(events, buf, null, 1.0, false, -1, updateScnIndex, 0);
    appender.run();
}

Example 12 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class TestDbusEventBufferAppendEvents method testAppendBigEventsHeadScnOverlap.

@Test
public /**
 * Test the case where we have a big event that overlaps both the DbusEventBuffer head and the
 * limit of the current buffer.
 */
void testAppendBigEventsHeadScnOverlap() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendBigEventsHeadScnOverlap");
    log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(7, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(222, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(0, tailIndexId);
    assertEquals(61, tailOffset);
    log.info("append windows with one small and one big event");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    generator.generateEvents(1, 1, 280, 139, events);
    generator.generateEvents(1, 1, 480, 339, events);
    // Add events to the EventBuffer. Now the buffer is full
    appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));
    headPos = dbusBuf.getHead();
    tailPos = dbusBuf.getTail();
    headGenId = parser.bufferGenId(headPos);
    headIndexId = parser.bufferIndex(headPos);
    headOffset = parser.bufferOffset(headPos);
    tailGenId = parser.bufferGenId(tailPos);
    tailIndexId = parser.bufferIndex(tailPos);
    tailOffset = parser.bufferOffset(tailPos);
    assertEquals(0, headGenId);
    assertEquals(2, headIndexId);
    assertEquals(61, headOffset);
    assertEquals(1, tailIndexId);
    assertEquals(461, tailOffset);
    assertEquals(322, dbusBuf.getBuffer()[0].limit());
    log.info("finished");
}

Example 13 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class ReadEventsTestParams method testScnIndexInvalidation.

@Test
public void testScnIndexInvalidation() throws Exception {
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 1000;
    int numScns = 10;
    int windowSize = numEvents / numScns;
    DbusEventGenerator evGen = new DbusEventGenerator();
    for (int i = 1; i < 2; ++i) {
        LOG.info("I=" + i);
        int bufferSize = (windowSize * 2);
        input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(bufferSize).setStagingBufferSize(bufferSize).setIndexSize(bufferSize).setIndividualBufferSize(bufferSize).setBatchSize(bufferSize).setProducerBufferSize(bufferSize).setPayloadSize(10).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
        DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
        if (srcTestEvents.isEmpty()) {
            assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), false, srcTestEvents) > 0);
        }
        int eventSize = 0;
        for (DbusEvent e : srcTestEvents) {
            if (!e.isControlMessage()) {
                eventSize = e.size();
                break;
            }
        }
        LOG.info("event size=" + eventSize);
        int indexSize = (input.getIndexSize() * eventSize / 100);
        LOG.info("indexSize=" + indexSize);
        DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize + i, input.getIndividualBufferSize() * eventSize, indexSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
        LOG.info("Allocated buffer size=" + prodEventBuffer.getAllocatedSize());
        DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats, 0.600);
        Thread t = new Thread(eventProducer);
        t.start();
        t.join();
        DbusEventsTotalStats stats = emitterStats.getTotalStats();
        LOG.info("minScn=" + prodEventBuffer.getMinScn() + "totalEvents=" + stats.getNumDataEvents());
    // weak assertion ! just check it's not -1 for now
    // TODO! get this assertion to succeed
    // assertTrue(prodEventBuffer.getMinScn()!=-1);
    }
}

Example 14 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class ReadEventsTestParams method appendGeneratedEvents.

public void appendGeneratedEvents() throws Exception {
    final DbusEventAppender appender = new DbusEventAppender(_srcEvents, _srcBuf, _srcBufStats);
    appender.run();
    Assert.assertEquals(_srcEvents.size(), _srcBufStats.getTotalStats().getNumDataEvents());
}

Example 15 with DbusEventAppender

use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.

the class ReadEventsTestParams method testStreamScn.

@Test
public void testStreamScn() throws Exception {
    // Src Event producer
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 500;
    int numScns = 10;
    int windowSize = numEvents / numScns;
    input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(numEvents * 2).setStagingBufferSize(numEvents * 2).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents * 2).setBatchSize(numEvents * 2).setProducerBufferSize(numEvents * 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
    input.setTestName("testStreamScn");
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1, "clientStats", true, true, null);
    DbusEventGenerator evGen = new DbusEventGenerator();
    assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) > 0);
    int eventSize = srcTestEvents.get(0).size();
    DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
    Vector<Long> seenScns = new Vector<Long>();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    boolean origEmptyValue = prodEventBuffer.empty();
    // empty buffer; prevScn=-1 , minScn=-1 ; so no Scn not found exception
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    boolean res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(-1L, prodEventBuffer.getPrevScn());
    assertTrue(res);
    assertEquals(0, seenScns.size());
    // partial buffer; with no complete window written;  prevScn > sinceScn , minScn=-1 ; Scn not found exception thrown;
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    assertEquals(-1L, prodEventBuffer.getMinScn());
    res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertFalse(res);
    assertEquals(0, seenScns.size());
    // restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);
    // partial buffer; with no complete window written;  sinceScn >= prevScn , minScn=-1 ; no exception;
    cp.setWindowScn(45L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertTrue(res);
    assertEquals(0, seenScns.size());
    // restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    long minScn = emitterStats.getTotalStats().getMinScn();
    long maxScn = emitterStats.getTotalStats().getMaxScn();
    long prevScn = emitterStats.getTotalStats().getPrevScn();
    System.out.printf("minScn=%d,maxScn=%d,prevScn=%d,range=%d\n", minScn, maxScn, prevScn, emitterStats.getTotalStats().getTimeSpan());
    assertEquals(numEvents - 1, emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS);
    assertEquals(prodEventBuffer.getTimestampOfFirstEvent(), emitterStats.getTotalStats().getTimestampMinScnEvent());
    // stream with scn < max; expect last window; not last 2
    cp.setWindowScn(maxScn - 1);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));
    // set windowScn to maxScn; get >= behaviour here ; get the last window
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(0);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));
    // stream with scn >= max ; get a window higher than max - expect nothing
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    // stream with scn > max
    cp.setWindowScn(maxScn + 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    // stream with scn >= min
    cp.setWindowScn(minScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(numScns - 1, seenScns.size());
    assertTrue(seenScns.get(0) != minScn);
    assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
    // stream with scn < min but >= prevScn
    cp.setWindowScn(prevScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns, clientStats);
    assertEquals(numScns, seenScns.size());
    assertEquals(Long.valueOf(minScn), seenScns.get(0));
    System.out.printf("Clientstats: minScn=%d , maxScn=%d , timespan=%d timeSinceLastEvent=%d\n", clientStats.getTotalStats().getMinScn(), clientStats.getTotalStats().getMaxScn(), clientStats.getTotalStats().getTimeSpan(), clientStats.getTotalStats().getTimeSinceLastEvent());
    assertEquals(maxScn, clientStats.getTotalStats().getMaxScn());
    assertEquals(numEvents - 1, clientStats.getTotalStats().getTimeSpan() / MILLISECONDS);
    assertEquals(clientStats.getTotalStats().getTimestampMaxScnEvent(), emitterStats.getTotalStats().getTimestampMaxScnEvent());
    assertEquals(minScn, clientStats.getTotalStats().getMinScn());
    // stream with scn < prevScn
    cp.setWindowScn(prevScn - 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    // stream with scn == prevScn but windowOffset=0
    cp.setWindowScn(prevScn);
    cp.setWindowOffset(0);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    // stream with scn > min
    cp.setWindowScn(minScn + 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(numScns - 1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
    assertTrue(seenScns.get(0) != minScn);
}