use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.
the class ReadEventsTestParams method testStatsMinMaxScn.
@Test
public void testStatsMinMaxScn() throws Exception {
// Src Event producer
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
EventBufferTestInput input = new EventBufferTestInput();
int numEvents = 10000;
long startScn = 1000;
int numWinScn = 10;
// set sharedBufferSize to a value much smaller than total size required
input.setNumEvents(numEvents).setWindowSize(numEvents / numWinScn).setSharedBufferSize(numEvents / 5).setStagingBufferSize(numEvents / 5).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents / 2).setBatchSize(numEvents / 5).setProducerBufferSize(numEvents / 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
input.setTestName("testStatsMinMaxScn");
DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
DbusEventGenerator evGen = new DbusEventGenerator(startScn);
if (evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) <= 0) {
fail();
return;
}
// sleep 10 ms;
int eventSize = srcTestEvents.get(0).size();
DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
tEmitter.join();
// sleep 10 ms;
int msDelay = 10;
Thread.sleep(msDelay);
long min = (numWinScn - 3) * input.getWindowSize() + startScn;
long max = numWinScn * input.getWindowSize() + startScn;
// note : event generator generates events such that a one second lag exists between the latest event and prev event
long expectedRange = (max - min) + input.getWindowSize() - 1;
System.out.printf("Total timespan = %d\n", (srcTestEvents.get(numEvents - 1).timestampInNanos() - srcTestEvents.get(0).timestampInNanos()) / NANOSECONDS);
System.out.printf("prevScn=%d\n", emitterStats.getTotalStats().getPrevScn());
System.out.printf("min = %d , max=%d buf=%d ,%d\n", emitterStats.getTotalStats().getMinScn(), emitterStats.getTotalStats().getMaxScn(), prodEventBuffer.getMinScn(), prodEventBuffer.lastWrittenScn());
System.out.printf("timespan=%d , timeSinceLastEvent = %d , timeSinceLastAccess %d\n", emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS, emitterStats.getTotalStats().getTimeSinceLastEvent(), emitterStats.getTotalStats().getTimeSinceLastAccess());
assertEquals(numEvents, srcTestEvents.size());
assertEquals(numEvents, emitterStats.getTotalStats().getNumDataEvents());
assertEquals(min, emitterStats.getTotalStats().getMinScn());
assertEquals(max, emitterStats.getTotalStats().getMaxScn());
assertEquals(min - input.getWindowSize(), emitterStats.getTotalStats().getPrevScn());
assertEquals(emitterStats.getTotalStats().getSizeDataEvents() * numEvents, numEvents * eventSize);
long tsSpanInSec = emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS;
assertEquals(expectedRange, tsSpanInSec);
long tsSinceLastEvent = emitterStats.getTotalStats().getTimeSinceLastEvent();
assertTrue(tsSinceLastEvent >= msDelay);
assertTrue(emitterStats.getTotalStats().getTimeLag() >= 0);
assertTrue(emitterStats.getTotalStats().getMinTimeLag() >= 0);
assertTrue(emitterStats.getTotalStats().getMaxTimeLag() >= 0);
DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(numEvents * 2L * eventSize, numEvents * eventSize, (numEvents / 10) * eventSize, numEvents * 2 * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
// check streaming
Vector<Long> seenScns = new Vector<Long>();
Checkpoint cp = new Checkpoint();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
// case: where sinceScn < min , > prevScn ; return the entire buffer
cp.setWindowScn(min - 1);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
int expectedNumWin = (int) (max - min) / input.getWindowSize() + 1;
assertEquals(expectedNumWin, seenScns.size());
assertEquals(Long.valueOf(min), seenScns.get(0));
assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
// case : where sinceScn < prevScn ; exception thrown;
cp.setWindowScn(startScn);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// case: where sinceScn < min , = prevScn ; offset=-1 ,return the entire buffer
cp.setWindowScn(prodEventBuffer.getPrevScn());
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(expectedNumWin, seenScns.size());
assertEquals(Long.valueOf(min), seenScns.get(0));
assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
// case: where sinceScn < min , = prevScn ; offset=0 ,return nothing
cp.setWindowScn(prodEventBuffer.getPrevScn());
cp.setWindowOffset(0);
seenScns.clear();
readEventBuffer.clear();
assertFalse(streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns));
assertEquals(0, seenScns.size());
}
use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.
the class ReadEventsTestParams method testReadEventsAssertSpanError.
/**
* Test to reproduce DDSDBUS-388.
*/
@Test
public void testReadEventsAssertSpanError() throws Exception {
final Logger log = Logger.getLogger("TestDbusEventBuffer.testReadEventsAssertSpanError");
// log.setLevel(Level.INFO);
log.info("start");
DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(10000, 10000, 320, 5000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
// dbusBuf.getLog().setLevel(Level.DEBUG);
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
// event size: 10 + 61 = 71
// window size: 5 * 71 + 61 = 416
// layout: [61 + 23 * 416 + 5 * 71 = 9984 ] <WrapAround> [ 61 + 22 * 416 + 2 * 71 + 61 = 9416 ]
// [ H=Gen:0,Ofs:0, T=Gen:0,Ofs:9984, L=10000 ]
// [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9416, L=9984 ]
generator.generateEvents(232, 5, 100, 10, events);
// Add events to the EventBuffer
DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
appender.run();
log.info("Dbus Event Buffer is :" + dbusBuf);
log.info("SCNIndex is :" + dbusBuf.getScnIndex());
final BufferPosition headPos = new BufferPosition(dbusBuf.getHead(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
final BufferPosition tailPos = new BufferPosition(dbusBuf.getTail(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
Assert.assertEquals(0, headPos.bufferGenId());
Assert.assertEquals(0, headPos.bufferIndex());
Assert.assertEquals(9629, headPos.bufferOffset());
Assert.assertEquals(1, tailPos.bufferGenId());
Assert.assertEquals(0, tailPos.bufferIndex());
Assert.assertEquals(9416, tailPos.bufferOffset());
Assert.assertEquals(9984, dbusBuf.getBuffer()[0].limit());
dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
final BufferPosition lastWinStart = new BufferPosition(61 + 22 * 416, dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
long lastScn = events.get(events.size() - 1).sequence();
generator = new DbusEventGenerator(lastScn - 1);
events.clear();
// event size: 10 + 61 = 71
generator.generateEvents(2, 2, 75, 10, events);
// intended layout: append 2 * 71 = 142 bytes
// [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9558, L=9984 ]
lastScn = events.get(events.size() - 1).sequence();
generator = new DbusEventGenerator(lastScn - 1);
// event size: 500 + 61 = 561
// partial window size: 561 = 622
generator.generateEvents(1, 1, 800, 500, events);
// intended layout: append 561 bytes;
// [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
// head moves to first window after 561: 61 + 2 * 416 = 893 > 561
// Set up the ReadChannel with new events
ByteArrayOutputStream oStream = new ByteArrayOutputStream();
WritableByteChannel oChannel = Channels.newChannel(oStream);
for (DbusEvent e : events) {
assertTrue("invalid event", e.isValid());
((DbusEventInternalReadable) e).writeTo(oChannel, Encoding.BINARY);
log.debug("event size is: " + e.size());
}
byte[] writeBytes = oStream.toByteArray();
ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
final ReadableByteChannel rChannel = Channels.newChannel(iStream);
dbusBuf.readEvents(rChannel);
log.info("Dbus Event Buffer is :" + dbusBuf);
log.info("SCNIndex is :" + dbusBuf.getScnIndex());
// expected layout:
// [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
headPos.setPosition(dbusBuf.getHead());
Assert.assertEquals(1, headPos.bufferGenId());
Assert.assertEquals(0, headPos.bufferIndex());
Assert.assertEquals(893, headPos.bufferOffset());
tailPos.setPosition(dbusBuf.getTail());
Assert.assertEquals(2, tailPos.bufferGenId());
Assert.assertEquals(0, tailPos.bufferIndex());
Assert.assertEquals(561, tailPos.bufferOffset());
Assert.assertEquals(9558, dbusBuf.getBuffer()[0].limit());
dbusBuf.getBufferPositionParser().assertSpan(dbusBuf.getHead(), dbusBuf.getTail(), true);
dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
// the second write did not add EOP so the SCN index tail should not have changed
dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
}
use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.
the class ReadEventsTestParams method testBasicRollback.
// Add a full window plus one event to the buffer, leaving room at the end, and then roll back.
// The write pointer should come back to the end of the event.
@Test
public void testBasicRollback() throws Exception {
// We have 7 byte buffers now, the last one having 200 bytes, others with 500 bytes.
DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(3200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, false);
events.clear();
// Same test case as before, with the first 3 byte buffers having data now.
// total event size = 499
generator.generateEvents(1, 20, 500, 438, events);
// total event size = 71
generator.generateEvents(1, 20, 500, 10, events);
// total event size = 428
generator.generateEvents(1, 20, 500, 367, events);
// event + EOP = 132.
generator.generateEvents(1, 1, 500, 10, events);
appender.run();
DbusEventBufferReflector reflector = appender.getDbusEventReflector();
// Now we should be able to add an event and roll it back as well.
events.clear();
// event size = 71, can append.
generator.generateEvents(1, 20, 500, 10, events);
long cwp1 = reflector.getCurrentWritePosition().getPosition();
dbusBuf.startEvents();
int evCount = 0;
evCount = appender.addEventToBuffer(events.firstElement(), evCount);
Assert.assertEquals(1, evCount);
long cwp2 = reflector.getCurrentWritePosition().getPosition();
Assert.assertTrue("Cwp should differ event write", cwp1 != cwp2);
// Roll back.
dbusBuf.rollbackEvents();
Assert.assertEquals(cwp1, reflector.getCurrentWritePosition().getPosition());
}
use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.
the class ReadEventsTestParams method testAppendEventOverlapNeq0.
/*
* This test-case is to recreate the bug where appendEvents incorrectly writes event without
* moving head.
*
* The following is the issue.
*
* head is at location x and tail is at location y
* The next event window is such that after adding "some" number of events in the window, both head
* and currentWritePosition is same (differing only in genId). Now adding the next event corrupts the
* buffer (and scnIndex) as head is not moved ahead.
*
*/
/**
* Recreate the head and tail position such that the eventBuffer is in the below state:
*
* <pre>
* NETBW
* |-----|
* ------------------------------------------------------------
* ^ ^ ^ ^
* | | | |
* 0 tail CWP,head capacity
*
* CWP : Current Write Position
* NETBW : Next Event to be written
* </pre>
*
* In this case, all pointers except head will be at (n+1)th generation while head
* is at nth generation.
*
* Two test cases are covered here:
* 1. n = 0
* 2. n > 0
*/
@Test
public void testAppendEventOverlapNeq0() throws Exception // Case n = 0;
{
final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
BufferPositionParser parser = dbusBuf.getBufferPositionParser();
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
generator.generateEvents(9, 3, 120, 39, events);
// Add events to the EventBuffer. Now the buffer is full
DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
// Logger.getRootLogger().setLevel(Level.ALL);
// running in the same thread
appender.run();
LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
long headPos = dbusBuf.getHead();
long tailPos = dbusBuf.getTail();
long scnIndexHead = dbusBuf.getScnIndex().getHead();
long scnIndexTail = dbusBuf.getScnIndex().getTail();
long headGenId = parser.bufferGenId(headPos);
long headIndexId = parser.bufferIndex(headPos);
long headOffset = parser.bufferOffset(headPos);
long tailGenId = parser.bufferGenId(tailPos);
long tailIndexId = parser.bufferIndex(tailPos);
long tailOffset = parser.bufferOffset(tailPos);
assertEquals("Head GenId", 0, headGenId);
assertEquals("Head Index", 0, headIndexId);
assertEquals("Head Offset", 0, headOffset);
assertEquals("Tail GenId", 0, tailGenId);
assertEquals("Tail Index", 0, tailIndexId);
assertEquals("Tail Offset", 1144, tailOffset);
assertEquals("SCNIndex Head", 0, scnIndexHead);
assertEquals("SCNIndex Tail", 80, scnIndexTail);
LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
events = new Vector<DbusEvent>();
generator = new DbusEventGenerator(100);
/*
* The event size is carefully created such that after adding 2nd
* event CWP and tail points to the same location. Now the 3rd event corrupts the EVB and index (in the presence of bug).
*/
generator.generateEvents(3, 2, 150, 89, events);
appender = new DbusEventAppender(events, dbusBuf, null);
// running in the same thread
appender.run();
LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
headPos = dbusBuf.getHead();
tailPos = dbusBuf.getTail();
headGenId = parser.bufferGenId(headPos);
headIndexId = parser.bufferIndex(headPos);
headOffset = parser.bufferOffset(headPos);
tailGenId = parser.bufferGenId(tailPos);
tailIndexId = parser.bufferIndex(tailPos);
tailOffset = parser.bufferOffset(tailPos);
scnIndexHead = dbusBuf.getScnIndex().getHead();
scnIndexTail = dbusBuf.getScnIndex().getTail();
assertEquals("Head GenId", 0, headGenId);
assertEquals("Head Index", 0, headIndexId);
assertEquals("Head Offset", 783, headOffset);
assertEquals("Tail GenId", 1, tailGenId);
assertEquals("Tail Index", 0, tailIndexId);
assertEquals("Tail Offset", 633, tailOffset);
assertEquals("SCNIndex Head", 64, scnIndexHead);
assertEquals("SCNIndex Tail", 48, scnIndexTail);
}
use of com.linkedin.databus.core.test.DbusEventAppender in project databus by linkedin.
the class ReadEventsTestParams method testAppendEventOverlapMany.
// Case where we dump lot of events while reaching the error case many times during this process.
@Test
public void testAppendEventOverlapMany() throws Exception {
final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1145, 5000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
BufferPositionParser parser = dbusBuf.getBufferPositionParser();
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
generator.generateEvents(9, 3, 120, 39, events);
// Add events to the EventBuffer. Now the buffer is full
DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
// running in the same thread
appender.run();
LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
long headPos = dbusBuf.getHead();
long tailPos = dbusBuf.getTail();
long scnIndexHead = dbusBuf.getScnIndex().getHead();
long scnIndexTail = dbusBuf.getScnIndex().getTail();
long headGenId = parser.bufferGenId(headPos);
long headIndexId = parser.bufferIndex(headPos);
long headOffset = parser.bufferOffset(headPos);
long tailGenId = parser.bufferGenId(tailPos);
long tailIndexId = parser.bufferIndex(tailPos);
long tailOffset = parser.bufferOffset(tailPos);
assertEquals("Head GenId", 0, headGenId);
assertEquals("Head Index", 0, headIndexId);
assertEquals("Head Offset", 0, headOffset);
assertEquals("Tail GenId", 0, tailGenId);
assertEquals("Tail Index", 0, tailIndexId);
assertEquals("Tail Offset", 1144, tailOffset);
assertEquals("SCNIndex Head", 0, scnIndexHead);
assertEquals("SCNIndex Tail", 80, scnIndexTail);
LOG.info("ScnIndex Head is :" + scnIndexHead + ", ScnIndex Tail is :" + scnIndexTail);
/*
* Dump lots of events
*/
events = new Vector<DbusEvent>();
generator = new DbusEventGenerator(100);
generator.generateEvents(655, 3, 150, 89, events);
appender = new DbusEventAppender(events, dbusBuf, null);
// running in the same thread
appender.run();
LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
headPos = dbusBuf.getHead();
tailPos = dbusBuf.getTail();
headGenId = parser.bufferGenId(headPos);
headIndexId = parser.bufferIndex(headPos);
headOffset = parser.bufferOffset(headPos);
tailGenId = parser.bufferGenId(tailPos);
tailIndexId = parser.bufferIndex(tailPos);
tailOffset = parser.bufferOffset(tailPos);
scnIndexHead = dbusBuf.getScnIndex().getHead();
scnIndexTail = dbusBuf.getScnIndex().getTail();
assertEquals("Head GenId", 109, headGenId);
assertEquals("Head Index", 0, headIndexId);
assertEquals("Head Offset", 511, headOffset);
assertEquals("Tail GenId", 110, tailGenId);
assertEquals("Tail Index", 0, tailIndexId);
assertEquals("Tail Offset", 211, tailOffset);
assertEquals("SCNIndex Head", 32, scnIndexHead);
assertEquals("SCNIndex Tail", 16, scnIndexTail);
/*
* The event size is carefully created such that after adding 2nd
* event CWP and tail points to the same location.
*/
events = new Vector<DbusEvent>();
generator = new DbusEventGenerator(10000);
generator.generateEvents(3, 5, 100, 28, events);
appender = new DbusEventAppender(events, dbusBuf, null);
// running in the same thread
appender.run();
LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
events = new Vector<DbusEvent>();
generator = new DbusEventGenerator(10000);
generator.generateEvents(3, 3, 120, 19, events);
headPos = dbusBuf.getHead();
tailPos = dbusBuf.getTail();
headGenId = parser.bufferGenId(headPos);
headIndexId = parser.bufferIndex(headPos);
headOffset = parser.bufferOffset(headPos);
tailGenId = parser.bufferGenId(tailPos);
tailIndexId = parser.bufferIndex(tailPos);
tailOffset = parser.bufferOffset(tailPos);
scnIndexHead = dbusBuf.getScnIndex().getHead();
scnIndexTail = dbusBuf.getScnIndex().getTail();
assertEquals("Head GenId", 110, headGenId);
assertEquals("Head Index", 0, headIndexId);
assertEquals("Head Offset", 0, headOffset);
assertEquals("Tail GenId", 110, tailGenId);
assertEquals("Tail Index", 0, tailIndexId);
assertEquals("Tail Offset", 600, tailOffset);
assertEquals("SCNIndex Head", 0, scnIndexHead);
assertEquals("SCNIndex Tail", 32, scnIndexTail);
}
Aggregations